diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 447873a595b..b155cfef302 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -244,6 +244,94 @@ GET /alias2/_search?q=user:kimchy&routing=2,3 // CONSOLE // TEST[continued] +[float] +[[aliases-write-index]] +==== Write Index + +It is possible to associate the index pointed to by an alias as the write index. +When specified, all index and update requests against an alias that point to multiple +indices will attempt to resolve to the one index that is the write index. +Only one index per alias can be assigned to be the write index at a time. If no write index is specified +and there are multiple indices referenced by an alias, then writes will not be allowed. + +It is possible to specify an index associated with an alias as a write index using both the aliases API +and index creation API. + +[source,js] +-------------------------------------------------- +POST /_aliases +{ + "actions" : [ + { + "add" : { + "index" : "test", + "alias" : "alias1", + "is_write_index" : true + } + } + ] +} +-------------------------------------------------- +// CONSOLE +// TEST[s/^/PUT test\n/] + +In this example, we associate the alias `alias1` to both `test` and `test2`, where +`test` will be the index chosen for writing to. + +[source,js] +-------------------------------------------------- +PUT /alias1/_doc/1 +{ + "foo": "bar" +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The new document that was indexed to `/alias1/_doc/1` will be indexed as if it were +`/test/_doc/1`. + +[source,js] +-------------------------------------------------- +GET /test/_doc/1 +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +To swap which index is the write index for an alias, the Aliases API can be leveraged to +do an atomic swap. The swap is not dependent on the ordering of the actions. + +[source,js] +-------------------------------------------------- +POST /_aliases +{ + "actions" : [ + { + "add" : { + "index" : "test", + "alias" : "alias1", + "is_write_index" : true + } + }, { + "add" : { + "index" : "test2", + "alias" : "alias1", + "is_write_index" : false + } + } + ] +} +-------------------------------------------------- +// CONSOLE +// TEST[s/^/PUT test\nPUT test2\n/] + +[IMPORTANT] +===================================== +Aliases that do not explicitly set `is_write_index: true` for an index, and +only reference one index, will have that referenced index behave as if it is the write index +until an additional index is referenced. At that point, there will be no write index and +writes will be rejected. +===================================== [float] [[alias-adding]] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index 8fafd9ef250..6f7c5a60093 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -79,7 +79,6 @@ indices.get_alias: index: test_index - - match: {test_index.aliases.test_alias: {}} - match: {test_index.aliases.test_blias.search_routing: b} - match: {test_index.aliases.test_blias.index_routing: b} - is_false: test_index.aliases.test_blias.filter @@ -87,6 +86,30 @@ - is_false: test_index.aliases.test_clias.index_routing - is_false: test_index.aliases.test_clias.search_routing +--- +"Create index with write aliases": + - skip: + version: " - 6.99.99" + reason: is_write_index is not implemented in ES <= 6.x + - do: + indices.create: + index: test_index + body: + aliases: + test_alias: {} + test_blias: + is_write_index: false + test_clias: + is_write_index: true + + - do: + indices.get_alias: + index: test_index + + - is_false: test_index.aliases.test_alias.is_write_index + - is_false: test_index.aliases.test_blias.is_write_index + - is_true: test_index.aliases.test_clias.is_write_index + --- "Create index with no type mappings": - do: diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java index 9172500a8cb..10ee8877fc9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -49,6 +50,7 @@ public class Alias implements Streamable, ToXContentFragment { private static final ParseField ROUTING = new ParseField("routing"); private static final ParseField INDEX_ROUTING = new ParseField("index_routing", "indexRouting", "index-routing"); private static final ParseField SEARCH_ROUTING = new ParseField("search_routing", "searchRouting", "search-routing"); + private static final ParseField IS_WRITE_INDEX = new ParseField("is_write_index"); private String name; @@ -61,6 +63,9 @@ public class Alias implements Streamable, ToXContentFragment { @Nullable private String searchRouting; + @Nullable + private Boolean writeIndex; + private Alias() { } @@ -167,6 +172,21 @@ public class Alias implements Streamable, ToXContentFragment { return this; } + /** + * @return the write index flag for the alias + */ + public Boolean writeIndex() { + return writeIndex; + } + + /** + * Sets whether an alias is pointing to a write-index + */ + public Alias writeIndex(@Nullable Boolean writeIndex) { + this.writeIndex = writeIndex; + return this; + } + /** * Allows to read an alias from the provided input stream */ @@ -182,6 +202,11 @@ public class Alias implements Streamable, ToXContentFragment { filter = in.readOptionalString(); indexRouting = in.readOptionalString(); searchRouting = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + writeIndex = in.readOptionalBoolean(); + } else { + writeIndex = null; + } } @Override @@ -190,6 +215,9 @@ public class Alias implements Streamable, ToXContentFragment { out.writeOptionalString(filter); out.writeOptionalString(indexRouting); out.writeOptionalString(searchRouting); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeOptionalBoolean(writeIndex); + } } /** @@ -219,6 +247,10 @@ public class Alias implements Streamable, ToXContentFragment { } else if (SEARCH_ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { alias.searchRouting(parser.text()); } + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + if (IS_WRITE_INDEX.match(currentFieldName, parser.getDeprecationHandler())) { + alias.writeIndex(parser.booleanValue()); + } } } return alias; @@ -245,6 +277,8 @@ public class Alias implements Streamable, ToXContentFragment { } } + builder.field(IS_WRITE_INDEX.getPreferredName(), writeIndex); + builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index 6332f50c145..00e3f7e32df 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -84,6 +85,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest REMOVE_PARSER = parser(REMOVE.getPreferredName(), AliasActions::remove); private static final ObjectParser REMOVE_INDEX_PARSER = parser(REMOVE_INDEX.getPreferredName(), @@ -215,6 +218,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(NAME, value.getStringRep()); } + + public static MaxAgeCondition fromXContent(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.VALUE_STRING) { + return new MaxAgeCondition(TimeValue.parseTimeValue(parser.text(), NAME)); + } else { + throw new IllegalArgumentException("invalid token: " + parser.currentToken()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java index 8fddb870e59..2f897fa6a01 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.rollover; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -61,4 +62,12 @@ public class MaxDocsCondition extends Condition { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(NAME, value); } + + public static MaxDocsCondition fromXContent(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.VALUE_NUMBER) { + return new MaxDocsCondition(parser.longValue()); + } else { + throw new IllegalArgumentException("invalid token: " + parser.currentToken()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java index bb6f37634ce..f1a121a87d4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -70,4 +71,12 @@ public class MaxSizeCondition extends Condition { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(NAME, value.getStringRep()); } + + public static MaxSizeCondition fromXContent(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.VALUE_STRING) { + return new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(parser.text(), NAME)); + } else { + throw new IllegalArgumentException("invalid token: " + parser.currentToken()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java new file mode 100644 index 00000000000..291dd3a0dda --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.rollover; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Class for holding Rollover related information within an index + */ +public class RolloverInfo extends AbstractDiffable implements Writeable, ToXContentFragment { + + public static final ParseField CONDITION_FIELD = new ParseField("met_conditions"); + public static final ParseField TIME_FIELD = new ParseField("time"); + + @SuppressWarnings("unchecked") + public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rollover_info", false, + (a, alias) -> new RolloverInfo(alias, (List) a[0], (Long) a[1])); + static { + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), + (p, c, n) -> p.namedObject(Condition.class, n, c), CONDITION_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), TIME_FIELD); + } + + private final String alias; + private final List metConditions; + private final long time; + + public RolloverInfo(String alias, List metConditions, long time) { + this.alias = alias; + this.metConditions = metConditions; + this.time = time; + } + + public RolloverInfo(StreamInput in) throws IOException { + this.alias = in.readString(); + this.time = in.readVLong(); + this.metConditions = in.readNamedWriteableList(Condition.class); + } + + public static RolloverInfo parse(XContentParser parser, String alias) { + return PARSER.apply(parser, alias); + } + + public String getAlias() { + return alias; + } + + public List getMetConditions() { + return metConditions; + } + + public long getTime() { + return time; + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(RolloverInfo::new, in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(alias); + out.writeVLong(time); + out.writeNamedWriteableList(metConditions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(alias); + builder.startObject(CONDITION_FIELD.getPreferredName()); + for (Condition condition : metConditions) { + condition.toXContent(builder, params); + } + builder.endObject(); + builder.field(TIME_FIELD.getPreferredName(), time); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(alias, metConditions, time); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + RolloverInfo other = (RolloverInfo) obj; + return Objects.equals(alias, other.alias) && + Objects.equals(metConditions, other.metConditions) && + Objects.equals(time, other.time); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index a5385c42aa0..c7780d41fab 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasAction; @@ -131,7 +132,9 @@ public class TransportRolloverAction extends TransportMasterNodeAction result)) { + List metConditions = rolloverRequest.getConditions().values().stream() + .filter(condition -> conditionResults.get(condition.toString())).collect(Collectors.toList()); + if (conditionResults.size() == 0 || metConditions.size() > 0) { CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName, rolloverRequest); createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> { @@ -141,13 +144,33 @@ public class TransportRolloverAction extends TransportMasterNodeAction { if (aliasClusterStateUpdateResponse.isAcknowledged()) { - activeShardsObserver.waitForActiveShards(new String[]{rolloverIndexName}, - rolloverRequest.getCreateIndexRequest().waitForActiveShards(), - rolloverRequest.masterNodeTimeout(), - isShardsAcknowledged -> listener.onResponse(new RolloverResponse( - sourceIndexName, rolloverIndexName, conditionResults, false, true, true, - isShardsAcknowledged)), - listener::onFailure); + clusterService.submitStateUpdateTask("update_rollover_info", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + RolloverInfo rolloverInfo = new RolloverInfo(rolloverRequest.getAlias(), metConditions, + threadPool.absoluteTimeInMillis()); + return ClusterState.builder(currentState) + .metaData(MetaData.builder(currentState.metaData()) + .put(IndexMetaData.builder(currentState.metaData().index(sourceIndexName)) + .putRolloverInfo(rolloverInfo))).build(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + activeShardsObserver.waitForActiveShards(new String[]{rolloverIndexName}, + rolloverRequest.getCreateIndexRequest().waitForActiveShards(), + rolloverRequest.masterNodeTimeout(), + isShardsAcknowledged -> listener.onResponse(new RolloverResponse( + sourceIndexName, rolloverIndexName, conditionResults, false, true, true, + isShardsAcknowledged)), + listener::onFailure); + } + }); } else { listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, false, false)); @@ -173,7 +196,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction actions = unmodifiableList(Arrays.asList( - new AliasAction.Add(newIndex, request.getAlias(), null, null, null), + new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null), new AliasAction.Remove(oldIndex, request.getAlias()))); final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(actions) .ackTimeout(request.ackTimeout()) diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 8ef6e4cd15b..ecdecc4457b 100644 --- a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; @@ -150,9 +151,11 @@ public abstract class TransportClient extends AbstractClient { SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter); SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class)); + IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); List entries = new ArrayList<>(); entries.addAll(NetworkModule.getNamedWriteables()); entries.addAll(searchModule.getNamedWriteables()); + entries.addAll(indicesModule.getNamedWriteables()); entries.addAll(ClusterModule.getNamedWriteables()); entries.addAll(pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.getNamedWriteables().stream()) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index ff49d072815..436ae79c103 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -51,7 +51,7 @@ public abstract class AliasAction { /** * Apply the action. - * + * * @param aliasValidator call to validate a new alias before adding it to the builder * @param metadata metadata builder for the changes made by all actions as part of this request * @param index metadata for the index being changed @@ -64,7 +64,7 @@ public abstract class AliasAction { */ @FunctionalInterface public interface NewAliasValidator { - void validate(String alias, @Nullable String indexRouting, @Nullable String filter); + void validate(String alias, @Nullable String indexRouting, @Nullable String filter, @Nullable Boolean writeIndex); } /** @@ -82,10 +82,14 @@ public abstract class AliasAction { @Nullable private final String searchRouting; + @Nullable + private final Boolean writeIndex; + /** * Build the operation. */ - public Add(String index, String alias, @Nullable String filter, @Nullable String indexRouting, @Nullable String searchRouting) { + public Add(String index, String alias, @Nullable String filter, @Nullable String indexRouting, + @Nullable String searchRouting, @Nullable Boolean writeIndex) { super(index); if (false == Strings.hasText(alias)) { throw new IllegalArgumentException("[alias] is required"); @@ -94,6 +98,7 @@ public abstract class AliasAction { this.filter = filter; this.indexRouting = indexRouting; this.searchRouting = searchRouting; + this.writeIndex = writeIndex; } /** @@ -103,6 +108,10 @@ public abstract class AliasAction { return alias; } + public Boolean writeIndex() { + return writeIndex; + } + @Override boolean removeIndex() { return false; @@ -110,15 +119,18 @@ public abstract class AliasAction { @Override boolean apply(NewAliasValidator aliasValidator, MetaData.Builder metadata, IndexMetaData index) { - aliasValidator.validate(alias, indexRouting, filter); + aliasValidator.validate(alias, indexRouting, filter, writeIndex); + AliasMetaData newAliasMd = AliasMetaData.newAliasMetaDataBuilder(alias).filter(filter).indexRouting(indexRouting) - .searchRouting(searchRouting).build(); + .searchRouting(searchRouting).writeIndex(writeIndex).build(); + // Check if this alias already exists AliasMetaData currentAliasMd = index.getAliases().get(alias); if (currentAliasMd != null && currentAliasMd.equals(newAliasMd)) { // It already exists, ignore it return false; } + metadata.put(IndexMetaData.builder(index).putAlias(newAliasMd)); return true; } @@ -182,4 +194,4 @@ public abstract class AliasAction { throw new UnsupportedOperationException(); } } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 945c94bcd64..29455123287 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -20,8 +20,10 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -55,7 +57,10 @@ public class AliasMetaData extends AbstractDiffable implements To private final Set searchRoutingValues; - private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting) { + @Nullable + private final Boolean writeIndex; + + private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -65,10 +70,11 @@ public class AliasMetaData extends AbstractDiffable implements To } else { searchRoutingValues = emptySet(); } + this.writeIndex = writeIndex; } private AliasMetaData(AliasMetaData aliasMetaData, String alias) { - this(alias, aliasMetaData.filter(), aliasMetaData.indexRouting(), aliasMetaData.searchRouting()); + this(alias, aliasMetaData.filter(), aliasMetaData.indexRouting(), aliasMetaData.searchRouting(), aliasMetaData.writeIndex()); } public String alias() { @@ -111,6 +117,10 @@ public class AliasMetaData extends AbstractDiffable implements To return searchRoutingValues; } + public Boolean writeIndex() { + return writeIndex; + } + public static Builder builder(String alias) { return new Builder(alias); } @@ -138,6 +148,8 @@ public class AliasMetaData extends AbstractDiffable implements To if (indexRouting != null ? !indexRouting.equals(that.indexRouting) : that.indexRouting != null) return false; if (searchRouting != null ? !searchRouting.equals(that.searchRouting) : that.searchRouting != null) return false; + if (writeIndex != null ? writeIndex != that.writeIndex : that.writeIndex != null) + return false; return true; } @@ -148,6 +160,7 @@ public class AliasMetaData extends AbstractDiffable implements To result = 31 * result + (filter != null ? filter.hashCode() : 0); result = 31 * result + (indexRouting != null ? indexRouting.hashCode() : 0); result = 31 * result + (searchRouting != null ? searchRouting.hashCode() : 0); + result = 31 * result + (writeIndex != null ? writeIndex.hashCode() : 0); return result; } @@ -173,6 +186,9 @@ public class AliasMetaData extends AbstractDiffable implements To out.writeBoolean(false); } + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeOptionalBoolean(writeIndex()); + } } public AliasMetaData(StreamInput in) throws IOException { @@ -194,6 +210,11 @@ public class AliasMetaData extends AbstractDiffable implements To searchRouting = null; searchRoutingValues = emptySet(); } + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + writeIndex = in.readOptionalBoolean(); + } else { + writeIndex = null; + } } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -221,6 +242,9 @@ public class AliasMetaData extends AbstractDiffable implements To private String searchRouting; + @Nullable + private Boolean writeIndex; + public Builder(String alias) { this.alias = alias; @@ -231,6 +255,7 @@ public class AliasMetaData extends AbstractDiffable implements To filter = aliasMetaData.filter(); indexRouting = aliasMetaData.indexRouting(); searchRouting = aliasMetaData.searchRouting(); + writeIndex = aliasMetaData.writeIndex(); } public String alias() { @@ -284,8 +309,13 @@ public class AliasMetaData extends AbstractDiffable implements To return this; } + public Builder writeIndex(@Nullable Boolean writeIndex) { + this.writeIndex = writeIndex; + return this; + } + public AliasMetaData build() { - return new AliasMetaData(alias, filter, indexRouting, searchRouting); + return new AliasMetaData(alias, filter, indexRouting, searchRouting, writeIndex); } public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -307,6 +337,10 @@ public class AliasMetaData extends AbstractDiffable implements To builder.field("search_routing", aliasMetaData.searchRouting()); } + if (aliasMetaData.writeIndex() != null) { + builder.field("is_write_index", aliasMetaData.writeIndex()); + } + builder.endObject(); } @@ -343,6 +377,10 @@ public class AliasMetaData extends AbstractDiffable implements To } } else if (token == XContentParser.Token.START_ARRAY) { parser.skipChildren(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + if ("is_write_index".equals(currentFieldName)) { + builder.writeIndex(parser.booleanValue()); + } } } return builder.build(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java index 786bd9af78a..d8bb04a1a39 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java @@ -19,12 +19,16 @@ package org.elasticsearch.cluster.metadata; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; /** * Encapsulates the {@link IndexMetaData} instances of a concrete index or indices an alias is pointing to. @@ -78,6 +82,7 @@ public interface AliasOrIndex { private final String aliasName; private final List referenceIndexMetaDatas; + private SetOnce writeIndex = new SetOnce<>(); public Alias(AliasMetaData aliasMetaData, IndexMetaData indexMetaData) { this.aliasName = aliasMetaData.getAlias(); @@ -90,11 +95,21 @@ public interface AliasOrIndex { return true; } + public String getAliasName() { + return aliasName; + } + @Override public List getIndices() { return referenceIndexMetaDatas; } + + @Nullable + public IndexMetaData getWriteIndex() { + return writeIndex.get(); + } + /** * Returns the unique alias metadata per concrete index. * @@ -138,5 +153,20 @@ public interface AliasOrIndex { this.referenceIndexMetaDatas.add(indexMetaData); } + public void computeAndValidateWriteIndex() { + List writeIndices = referenceIndexMetaDatas.stream() + .filter(idxMeta -> Boolean.TRUE.equals(idxMeta.getAliases().get(aliasName).writeIndex())) + .collect(Collectors.toList()); + if (referenceIndexMetaDatas.size() == 1) { + writeIndex.set(referenceIndexMetaDatas.get(0)); + } else if (writeIndices.size() == 1) { + writeIndex.set(writeIndices.get(0)); + } else if (writeIndices.size() > 1) { + List writeIndicesStrings = writeIndices.stream() + .map(i -> i.getIndex().getName()).collect(Collectors.toList()); + throw new IllegalStateException("alias [" + aliasName + "] has more than one write index [" + + Strings.collectionToCommaDelimitedString(writeIndicesStrings) + "]"); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index 7f1348dd158..33e1687e241 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -57,7 +57,7 @@ public class AliasValidator extends AbstractComponent { * @throws IllegalArgumentException if the alias is not valid */ public void validateAlias(Alias alias, String index, MetaData metaData) { - validateAlias(alias.name(), index, alias.indexRouting(), name -> metaData.index(name)); + validateAlias(alias.name(), index, alias.indexRouting(), metaData::index); } /** @@ -66,7 +66,7 @@ public class AliasValidator extends AbstractComponent { * @throws IllegalArgumentException if the alias is not valid */ public void validateAliasMetaData(AliasMetaData aliasMetaData, String index, MetaData metaData) { - validateAlias(aliasMetaData.alias(), index, aliasMetaData.indexRouting(), name -> metaData.index(name)); + validateAlias(aliasMetaData.alias(), index, aliasMetaData.indexRouting(), metaData::index); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index c871dd61830..9e4f8497878 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -25,6 +25,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; @@ -294,6 +295,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen static final String KEY_STATE = "state"; static final String KEY_MAPPINGS = "mappings"; static final String KEY_ALIASES = "aliases"; + static final String KEY_ROLLOVER_INFOS = "rollover_info"; public static final String KEY_PRIMARY_TERMS = "primary_terms"; public static final String INDEX_STATE_FILE_PREFIX = "state-"; @@ -331,13 +333,14 @@ public class IndexMetaData implements Diffable, ToXContentFragmen private final Version indexUpgradedVersion; private final ActiveShardCount waitForActiveShards; + private final ImmutableOpenMap rolloverInfos; private IndexMetaData(Index index, long version, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs, ImmutableOpenIntMap> inSyncAllocationIds, DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters, Version indexCreatedVersion, Version indexUpgradedVersion, - int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) { + int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards, ImmutableOpenMap rolloverInfos) { this.index = index; this.version = version; @@ -362,6 +365,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen this.routingFactor = routingNumShards / numberOfShards; this.routingPartitionSize = routingPartitionSize; this.waitForActiveShards = waitForActiveShards; + this.rolloverInfos = rolloverInfos; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; } @@ -517,6 +521,10 @@ public class IndexMetaData implements Diffable, ToXContentFragmen return inSyncAllocationIds; } + public ImmutableOpenMap getRolloverInfos() { + return rolloverInfos; + } + public Set inSyncAllocationIds(int shardId) { assert shardId >= 0 && shardId < numberOfShards; return inSyncAllocationIds.get(shardId); @@ -587,6 +595,9 @@ public class IndexMetaData implements Diffable, ToXContentFragmen if (!inSyncAllocationIds.equals(that.inSyncAllocationIds)) { return false; } + if (rolloverInfos.equals(that.rolloverInfos) == false) { + return false; + } return true; } @@ -603,6 +614,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen result = 31 * result + Long.hashCode(routingNumShards); result = 31 * result + Arrays.hashCode(primaryTerms); result = 31 * result + inSyncAllocationIds.hashCode(); + result = 31 * result + rolloverInfos.hashCode(); return result; } @@ -638,6 +650,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen private final Diff> aliases; private final Diff> customs; private final Diff>> inSyncAllocationIds; + private final Diff> rolloverInfos; IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) { index = after.index.getName(); @@ -651,6 +664,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer()); inSyncAllocationIds = DiffableUtils.diff(before.inSyncAllocationIds, after.inSyncAllocationIds, DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); + rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer()); } IndexMetaDataDiff(StreamInput in) throws IOException { @@ -679,6 +693,13 @@ public class IndexMetaData implements Diffable, ToXContentFragmen }); inSyncAllocationIds = DiffableUtils.readImmutableOpenIntMapDiff(in, DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + rolloverInfos = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), RolloverInfo::new, + RolloverInfo::readDiffFrom); + } else { + ImmutableOpenMap emptyMap = ImmutableOpenMap.of(); + rolloverInfos = DiffableUtils.diff(emptyMap, emptyMap, DiffableUtils.getStringKeySerializer()); + } } @Override @@ -693,6 +714,9 @@ public class IndexMetaData implements Diffable, ToXContentFragmen aliases.writeTo(out); customs.writeTo(out); inSyncAllocationIds.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + rolloverInfos.writeTo(out); + } } @Override @@ -707,6 +731,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen builder.aliases.putAll(aliases.apply(part.aliases)); builder.customs.putAll(customs.apply(part.customs)); builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds)); + builder.rolloverInfos.putAll(rolloverInfos.apply(part.rolloverInfos)); return builder.build(); } } @@ -740,6 +765,12 @@ public class IndexMetaData implements Diffable, ToXContentFragmen Set allocationIds = DiffableUtils.StringSetValueSerializer.getInstance().read(in, key); builder.putInSyncAllocationIds(key, allocationIds); } + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + int rolloverAliasesSize = in.readVInt(); + for (int i = 0; i < rolloverAliasesSize; i++) { + builder.putRolloverInfo(new RolloverInfo(in)); + } + } return builder.build(); } @@ -769,6 +800,12 @@ public class IndexMetaData implements Diffable, ToXContentFragmen out.writeVInt(cursor.key); DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out); } + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeVInt(rolloverInfos.size()); + for (ObjectCursor cursor : rolloverInfos.values()) { + cursor.value.writeTo(out); + } + } } public static Builder builder(String index) { @@ -790,6 +827,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen private final ImmutableOpenMap.Builder aliases; private final ImmutableOpenMap.Builder customs; private final ImmutableOpenIntMap.Builder> inSyncAllocationIds; + private final ImmutableOpenMap.Builder rolloverInfos; private Integer routingNumShards; public Builder(String index) { @@ -798,6 +836,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen this.aliases = ImmutableOpenMap.builder(); this.customs = ImmutableOpenMap.builder(); this.inSyncAllocationIds = ImmutableOpenIntMap.builder(); + this.rolloverInfos = ImmutableOpenMap.builder(); } public Builder(IndexMetaData indexMetaData) { @@ -811,6 +850,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen this.customs = ImmutableOpenMap.builder(indexMetaData.customs); this.routingNumShards = indexMetaData.routingNumShards; this.inSyncAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.inSyncAllocationIds); + this.rolloverInfos = ImmutableOpenMap.builder(indexMetaData.rolloverInfos); } public String index() { @@ -951,6 +991,15 @@ public class IndexMetaData implements Diffable, ToXContentFragmen return this; } + public RolloverInfo getRolloverInfo(String alias) { + return rolloverInfos.get(alias); + } + + public Builder putRolloverInfo(RolloverInfo rolloverInfo) { + rolloverInfos.put(rolloverInfo.getAlias(), rolloverInfo); + return this; + } + public long version() { return this.version; } @@ -1089,7 +1138,7 @@ public class IndexMetaData implements Diffable, ToXContentFragmen return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters, - indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards); + indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, rolloverInfos.build()); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -1143,6 +1192,12 @@ public class IndexMetaData implements Diffable, ToXContentFragmen } builder.endObject(); + builder.startObject(KEY_ROLLOVER_INFOS); + for (ObjectCursor cursor : indexMetaData.getRolloverInfos().values()) { + cursor.value.toXContent(builder, params); + } + builder.endObject(); + builder.endObject(); } @@ -1202,6 +1257,16 @@ public class IndexMetaData implements Diffable, ToXContentFragmen throw new IllegalArgumentException("Unexpected token: " + token); } } + } else if (KEY_ROLLOVER_INFOS.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + builder.putRolloverInfo(RolloverInfo.parse(parser, currentFieldName)); + } else { + throw new IllegalArgumentException("Unexpected token: " + token); + } + } } else if ("warmers".equals(currentFieldName)) { // TODO: do this in 6.0: // throw new IllegalArgumentException("Warmers are not supported anymore - are you upgrading from 1.x?"); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 5657873987e..2bfe0d0a58f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -1039,7 +1039,22 @@ public class MetaData implements Iterable, Diffable, To } - // build all indices map + SortedMap aliasAndIndexLookup = Collections.unmodifiableSortedMap(buildAliasAndIndexLookup()); + + + // build all concrete indices arrays: + // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices. + // When doing an operation across all indices, most of the time is spent on actually going to all shards and + // do the required operations, the bottleneck isn't resolving expressions into concrete indices. + String[] allIndicesArray = allIndices.toArray(new String[allIndices.size()]); + String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]); + String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]); + + return new MetaData(clusterUUID, version, transientSettings, persistentSettings, indices.build(), templates.build(), + customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, aliasAndIndexLookup); + } + + private SortedMap buildAliasAndIndexLookup() { SortedMap aliasAndIndexLookup = new TreeMap<>(); for (ObjectCursor cursor : indices.values()) { IndexMetaData indexMetaData = cursor.value; @@ -1059,17 +1074,9 @@ public class MetaData implements Iterable, Diffable, To }); } } - aliasAndIndexLookup = Collections.unmodifiableSortedMap(aliasAndIndexLookup); - // build all concrete indices arrays: - // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices. - // When doing an operation across all indices, most of the time is spent on actually going to all shards and - // do the required operations, the bottleneck isn't resolving expressions into concrete indices. - String[] allIndicesArray = allIndices.toArray(new String[allIndices.size()]); - String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]); - String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]); - - return new MetaData(clusterUUID, version, transientSettings, persistentSettings, indices.build(), templates.build(), - customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, aliasAndIndexLookup); + aliasAndIndexLookup.values().stream().filter(AliasOrIndex::isAlias) + .forEach(alias -> ((AliasOrIndex.Alias) alias).computeAndValidateWriteIndex()); + return aliasAndIndexLookup; } public static String toXContent(MetaData metaData) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 0d8a374e66d..be9db5262b0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -516,7 +516,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { } for (Alias alias : request.aliases()) { AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter()) - .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build(); + .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).writeIndex(alias.writeIndex()).build(); indexMetaDataBuilder.putAlias(aliasMetaData); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java index 0c38371bdc9..28dc7f2425d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java @@ -127,7 +127,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent { if (index == null) { throw new IndexNotFoundException(action.getIndex()); } - NewAliasValidator newAliasValidator = (alias, indexRouting, filter) -> { + NewAliasValidator newAliasValidator = (alias, indexRouting, filter, writeIndex) -> { /* It is important that we look up the index using the metadata builder we are modifying so we can remove an * index and replace it with an alias. */ Function indexLookup = name -> metadata.get(name); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 11024286b22..2d41491e3a7 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -24,9 +24,12 @@ import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.BinaryFieldMapper; @@ -62,6 +65,7 @@ import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.plugins.MapperPlugin; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; @@ -86,15 +90,26 @@ public class IndicesModule extends AbstractModule { } private void registerBuiltinWritables() { - namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new)); - namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new)); - namedWritables.add(new Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); } - public List getNamedWriteables() { + public List getNamedWriteables() { return namedWritables; } + public List getNamedXContents() { + return Arrays.asList( + new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxAgeCondition.NAME), (p, c) -> + MaxAgeCondition.fromXContent(p)), + new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxDocsCondition.NAME), (p, c) -> + MaxDocsCondition.fromXContent(p)), + new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxSizeCondition.NAME), (p, c) -> + MaxSizeCondition.fromXContent(p)) + ); + } + private Map getMappers(List mapperPlugins) { Map mappers = new LinkedHashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index b6274223d22..aa9b3943e88 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -389,6 +389,7 @@ public class Node implements Closeable { final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of( NetworkModule.getNamedXContents().stream(), + indicesModule.getNamedXContents().stream(), searchModule.getNamedXContents().stream(), pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.getNamedXContent().stream()), diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java index 202530ccf92..f2ae67e1fc1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java @@ -114,6 +114,7 @@ public class AliasActionsTests extends ESTestCase { Map filter = randomBoolean() ? randomMap(5) : null; Object searchRouting = randomBoolean() ? randomRouting() : null; Object indexRouting = randomBoolean() ? randomBoolean() ? searchRouting : randomRouting() : null; + boolean writeIndex = randomBoolean(); XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent()); b.startObject(); { @@ -142,6 +143,7 @@ public class AliasActionsTests extends ESTestCase { if (indexRouting != null && false == indexRouting.equals(searchRouting)) { b.field("index_routing", indexRouting); } + b.field("is_write_index", writeIndex); } b.endObject(); } @@ -159,6 +161,7 @@ public class AliasActionsTests extends ESTestCase { } assertEquals(Objects.toString(searchRouting, null), action.searchRouting()); assertEquals(Objects.toString(indexRouting, null), action.indexRouting()); + assertEquals(writeIndex, action.writeIndex()); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java index dbca9f7a98f..e50805ab5b2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -93,6 +93,7 @@ public class CreateIndexRequestTests extends ESTestCase { Alias alias = new Alias("test_alias"); alias.routing("1"); alias.filter("{\"term\":{\"year\":2016}}"); + alias.writeIndex(true); request.alias(alias); Settings.Builder settings = Settings.builder(); @@ -103,7 +104,7 @@ public class CreateIndexRequestTests extends ESTestCase { String expectedRequestBody = "{\"settings\":{\"index\":{\"number_of_shards\":\"10\"}}," + "\"mappings\":{\"my_type\":{\"type\":{}}}," + - "\"aliases\":{\"test_alias\":{\"filter\":{\"term\":{\"year\":2016}},\"routing\":\"1\"}}}"; + "\"aliases\":{\"test_alias\":{\"filter\":{\"term\":{\"year\":2016}},\"routing\":\"1\",\"is_write_index\":true}}}"; assertEquals(expectedRequestBody, actualRequestBody); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java index 869bba452fe..aa35d9d273a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java @@ -37,6 +37,7 @@ import org.joda.time.format.DateTimeFormat; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Set; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -44,6 +45,10 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.CombinableMatcher.both; +import static org.hamcrest.number.OrderingComparison.greaterThanOrEqualTo; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RolloverIT extends ESIntegTestCase { @@ -70,6 +75,7 @@ public class RolloverIT extends ESIntegTestCase { } public void testRollover() throws Exception { + long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L; assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias")).get()); index("test_index-2", "type1", "1", "field", "value"); flush("test_index-2"); @@ -84,6 +90,11 @@ public class RolloverIT extends ESIntegTestCase { assertFalse(oldIndex.getAliases().containsKey("test_alias")); final IndexMetaData newIndex = state.metaData().index("test_index-000003"); assertTrue(newIndex.getAliases().containsKey("test_alias")); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(1)); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getAlias(), equalTo("test_alias")); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getMetConditions(), is(empty())); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(), + is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L)))); } public void testRolloverWithIndexSettings() throws Exception { @@ -246,17 +257,27 @@ public class RolloverIT extends ESIntegTestCase { assertThat(response.getOldIndex(), equalTo("test-1")); assertThat(response.getNewIndex(), equalTo("test-000002")); assertThat("No rollover with a large max_size condition", response.isRolledOver(), equalTo(false)); + final IndexMetaData oldIndex = client().admin().cluster().prepareState().get().getState().metaData().index("test-1"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); } // A small max_size { + ByteSizeValue maxSizeValue = new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES); + long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L; final RolloverResponse response = client().admin().indices() .prepareRolloverIndex("test_alias") - .addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES)) + .addMaxIndexSizeCondition(maxSizeValue) .get(); assertThat(response.getOldIndex(), equalTo("test-1")); assertThat(response.getNewIndex(), equalTo("test-000002")); assertThat("Should rollover with a small max_size condition", response.isRolledOver(), equalTo(true)); + final IndexMetaData oldIndex = client().admin().cluster().prepareState().get().getState().metaData().index("test-1"); + List metConditions = oldIndex.getRolloverInfos().get("test_alias").getMetConditions(); + assertThat(metConditions.size(), equalTo(1)); + assertThat(metConditions.get(0).toString(), equalTo(new MaxSizeCondition(maxSizeValue).toString())); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(), + is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L)))); } // An empty index @@ -268,6 +289,8 @@ public class RolloverIT extends ESIntegTestCase { assertThat(response.getOldIndex(), equalTo("test-000002")); assertThat(response.getNewIndex(), equalTo("test-000003")); assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false)); + final IndexMetaData oldIndex = client().admin().cluster().prepareState().get().getState().metaData().index("test-000002"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java index ba595de5215..4fa99374f0f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -71,6 +71,7 @@ public class ResizeRequestTests extends ESTestCase { Alias alias = new Alias("test_alias"); alias.routing("1"); alias.filter("{\"term\":{\"year\":2016}}"); + alias.writeIndex(true); target.alias(alias); Settings.Builder settings = Settings.builder(); settings.put(SETTING_NUMBER_OF_SHARDS, 10); @@ -78,7 +79,7 @@ public class ResizeRequestTests extends ESTestCase { request.setTargetIndex(target); String actualRequestBody = Strings.toString(request); String expectedRequestBody = "{\"settings\":{\"index\":{\"number_of_shards\":\"10\"}}," + - "\"aliases\":{\"test_alias\":{\"filter\":{\"term\":{\"year\":2016}},\"routing\":\"1\"}}}"; + "\"aliases\":{\"test_alias\":{\"filter\":{\"term\":{\"year\":2016}},\"routing\":\"1\",\"is_write_index\":true}}}"; assertEquals(expectedRequestBody, actualRequestBody); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java index 00865cc9a65..de23c560eb9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java @@ -41,6 +41,7 @@ public class AliasMetaDataTests extends AbstractXContentTestCase .indexRouting("indexRouting") .routing("routing") .searchRouting("trim,tw , ltw , lw") + .writeIndex(randomBoolean() ? null : randomBoolean()) .build(); assertThat(before.searchRoutingValues(), equalTo(Sets.newHashSet("trim", "tw ", " ltw ", " lw"))); @@ -54,6 +55,21 @@ public class AliasMetaDataTests extends AbstractXContentTestCase assertThat(after, equalTo(before)); } + @Override + protected void assertEqualInstances(AliasMetaData expectedInstance, AliasMetaData newInstance) { + assertNotSame(newInstance, expectedInstance); + if (expectedInstance.writeIndex() == null) { + expectedInstance = AliasMetaData.builder(expectedInstance.alias()) + .filter(expectedInstance.filter()) + .indexRouting(expectedInstance.indexRouting()) + .searchRouting(expectedInstance.searchRouting()) + .writeIndex(randomBoolean() ? null : randomBoolean()) + .build(); + } + assertEquals(expectedInstance, newInstance); + assertEquals(expectedInstance.hashCode(), newInstance.hashCode()); + } + @Override protected AliasMetaData createTestInstance() { return createTestItem(); @@ -95,6 +111,7 @@ public class AliasMetaDataTests extends AbstractXContentTestCase if (randomBoolean()) { builder.filter("{\"term\":{\"year\":2016}}"); } + builder.writeIndex(randomBoolean()); return builder.build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 56fbf1db245..744a29e843c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -69,6 +69,7 @@ import static org.elasticsearch.test.hamcrest.CollectionAssertions.hasKey; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -291,6 +292,32 @@ public class IndexCreationTaskTests extends ESTestCase { assertThat(e.getMessage(), containsString("invalid wait_for_active_shards")); } + public void testWriteIndex() throws Exception { + Boolean writeIndex = randomBoolean() ? null : randomBoolean(); + setupRequestAlias(new Alias("alias1").writeIndex(writeIndex)); + setupRequestMapping("mapping1", createMapping()); + setupRequestCustom("custom1", createCustom()); + reqSettings.put("key1", "value1"); + + final ClusterState result = executeTask(); + assertThat(result.metaData().index("test").getAliases(), hasKey("alias1")); + assertThat(result.metaData().index("test").getAliases().get("alias1").writeIndex(), equalTo(writeIndex)); + } + + public void testWriteIndexValidationException() throws Exception { + IndexMetaData existingWriteIndex = IndexMetaData.builder("test2") + .settings(settings(Version.CURRENT)).putAlias(AliasMetaData.builder("alias1").writeIndex(true).build()) + .numberOfShards(1).numberOfReplicas(0).build(); + idxBuilder.put("test2", existingWriteIndex); + setupRequestMapping("mapping1", createMapping()); + setupRequestCustom("custom1", createCustom()); + reqSettings.put("key1", "value1"); + setupRequestAlias(new Alias("alias1").writeIndex(true)); + + Exception exception = expectThrows(IllegalStateException.class, () -> executeTask()); + assertThat(exception.getMessage(), startsWith("alias [alias1] has more than one write index [")); + } + private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) { final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 7734a9d7b4e..9e8a5e04f43 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -19,18 +19,31 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.Set; @@ -38,6 +51,23 @@ import static org.hamcrest.Matchers.is; public class IndexMetaDataTests extends ESTestCase { + private IndicesModule INDICES_MODULE = new IndicesModule(Collections.emptyList()); + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + protected NamedWriteableRegistry writableRegistry() { + return new NamedWriteableRegistry(INDICES_MODULE.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(INDICES_MODULE.getNamedXContents()); + } + public void testIndexMetaDataSerialization() throws IOException { Integer numShard = randomFrom(1, 2, 4, 8, 16); int numberOfReplicas = randomIntBetween(0, 10); @@ -50,7 +80,12 @@ public class IndexMetaDataTests extends ESTestCase { .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) - .build(); + .putRolloverInfo( + new RolloverInfo(randomAlphaOfLength(5), + Arrays.asList(new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong())), + randomNonNegativeLong())).build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -71,17 +106,20 @@ public class IndexMetaDataTests extends ESTestCase { final BytesStreamOutput out = new BytesStreamOutput(); metaData.writeTo(out); - IndexMetaData deserialized = IndexMetaData.readFrom(out.bytes().streamInput()); - assertEquals(metaData, deserialized); - assertEquals(metaData.hashCode(), deserialized.hashCode()); + try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry())) { + IndexMetaData deserialized = IndexMetaData.readFrom(in); + assertEquals(metaData, deserialized); + assertEquals(metaData.hashCode(), deserialized.hashCode()); - assertEquals(metaData.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); - assertEquals(metaData.getNumberOfShards(), deserialized.getNumberOfShards()); - assertEquals(metaData.getCreationVersion(), deserialized.getCreationVersion()); - assertEquals(metaData.getRoutingNumShards(), deserialized.getRoutingNumShards()); - assertEquals(metaData.getCreationDate(), deserialized.getCreationDate()); - assertEquals(metaData.getRoutingFactor(), deserialized.getRoutingFactor()); - assertEquals(metaData.primaryTerm(0), deserialized.primaryTerm(0)); + assertEquals(metaData.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); + assertEquals(metaData.getNumberOfShards(), deserialized.getNumberOfShards()); + assertEquals(metaData.getCreationVersion(), deserialized.getCreationVersion()); + assertEquals(metaData.getRoutingNumShards(), deserialized.getRoutingNumShards()); + assertEquals(metaData.getCreationDate(), deserialized.getCreationDate()); + assertEquals(metaData.getRoutingFactor(), deserialized.getRoutingFactor()); + assertEquals(metaData.primaryTerm(0), deserialized.primaryTerm(0)); + assertEquals(metaData.getRolloverInfos(), deserialized.getRolloverInfos()); + } } public void testGetRoutingFactor() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java index e5b52d8cf52..812dfd8f6f6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; @@ -29,9 +30,13 @@ import org.elasticsearch.test.VersionUtils; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anySetOf; import static org.mockito.Mockito.mock; @@ -64,7 +69,7 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase { ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index); // Add an alias to it - ClusterState after = service.innerExecute(before, singletonList(new AliasAction.Add(index, "test", null, null, null))); + ClusterState after = service.innerExecute(before, singletonList(new AliasAction.Add(index, "test", null, null, null, null))); AliasOrIndex alias = after.metaData().getAliasAndIndexLookup().get("test"); assertNotNull(alias); assertTrue(alias.isAlias()); @@ -74,7 +79,7 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase { before = after; after = service.innerExecute(before, Arrays.asList( new AliasAction.Remove(index, "test"), - new AliasAction.Add(index, "test_2", null, null, null))); + new AliasAction.Add(index, "test_2", null, null, null, null))); assertNull(after.metaData().getAliasAndIndexLookup().get("test")); alias = after.metaData().getAliasAndIndexLookup().get("test_2"); assertNotNull(alias); @@ -95,7 +100,7 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase { // Now remove "test" and add an alias to "test" to "test_2" in one go ClusterState after = service.innerExecute(before, Arrays.asList( - new AliasAction.Add("test_2", "test", null, null, null), + new AliasAction.Add("test_2", "test", null, null, null, null), new AliasAction.RemoveIndex("test"))); AliasOrIndex alias = after.metaData().getAliasAndIndexLookup().get("test"); assertNotNull(alias); @@ -109,7 +114,7 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase { // Attempt to add an alias to "test" at the same time as we remove it IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> service.innerExecute(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null), + new AliasAction.Add("test", "alias", null, null, null, null), new AliasAction.RemoveIndex("test")))); assertEquals("test", e.getIndex().getName()); } @@ -125,6 +130,127 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase { assertNull(after.metaData().getAliasAndIndexLookup().get("test")); } + public void testAddWriteOnlyWithNoExistingAliases() { + ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), "test"); + + ClusterState after = service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, false))); + assertFalse(after.metaData().index("test").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test"))); + + after = service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, null))); + assertNull(after.metaData().index("test").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test"))); + + after = service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, true))); + assertTrue(after.metaData().index("test").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test"))); + } + + public void testAddWriteOnlyWithExistingWriteIndex() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .putAlias(AliasMetaData.builder("alias").writeIndex(true).build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); + + ClusterState after = service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, null))); + assertNull(after.metaData().index("test").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test2"))); + + Exception exception = expectThrows(IllegalStateException.class, () -> service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, true)))); + assertThat(exception.getMessage(), startsWith("alias [alias] has more than one write index [")); + } + + public void testSwapWriteOnlyIndex() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .putAlias(AliasMetaData.builder("alias").writeIndex(true).build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); + + Boolean unsetValue = randomBoolean() ? null : false; + List swapActions = Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, unsetValue), + new AliasAction.Add("test2", "alias", null, null, null, true) + ); + Collections.shuffle(swapActions, random()); + ClusterState after = service.innerExecute(before, swapActions); + assertThat(after.metaData().index("test").getAliases().get("alias").writeIndex(), equalTo(unsetValue)); + assertTrue(after.metaData().index("test2").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test2"))); + } + + public void testAddWriteOnlyWithExistingNonWriteIndices() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .putAlias(AliasMetaData.builder("alias").writeIndex(randomBoolean() ? null : false).build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .putAlias(AliasMetaData.builder("alias").writeIndex(randomBoolean() ? null : false).build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData3 = IndexMetaData.builder("test3") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2).put(indexMetaData3)).build(); + + assertNull(((AliasOrIndex.Alias) before.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex()); + + ClusterState after = service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test3", "alias", null, null, null, true))); + assertTrue(after.metaData().index("test3").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test3"))); + + } + + public void testAddWriteOnlyWithIndexRemoved() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .putAlias(AliasMetaData.builder("alias").build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .putAlias(AliasMetaData.builder("alias").build()) + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); + + assertNull(before.metaData().index("test").getAliases().get("alias").writeIndex()); + assertNull(before.metaData().index("test2").getAliases().get("alias").writeIndex()); + assertNull(((AliasOrIndex.Alias) before.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex()); + + ClusterState after = service.innerExecute(before, Collections.singletonList(new AliasAction.RemoveIndex("test"))); + assertNull(after.metaData().index("test2").getAliases().get("alias").writeIndex()); + assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), + equalTo(after.metaData().index("test2"))); + } + + public void testAddWriteOnlyValidatesAgainstMetaDataBuilder() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); + + Exception exception = expectThrows(IllegalStateException.class, () -> service.innerExecute(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, true), + new AliasAction.Add("test2", "alias", null, null, null, true) + ))); + assertThat(exception.getMessage(), startsWith("alias [alias] has more than one write index [")); + } + private ClusterState createIndex(ClusterState state, String index) { IndexMetaData indexMetaData = IndexMetaData.builder(index) .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 3a83580dc1c..96a533118c8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -99,6 +99,34 @@ public class MetaDataTests extends ESTestCase { } } + public void testValidateAliasWriteOnly() { + String alias = randomAlphaOfLength(5); + String indexA = randomAlphaOfLength(6); + String indexB = randomAlphaOfLength(7); + Boolean aWriteIndex = randomBoolean() ? null : randomBoolean(); + Boolean bWriteIndex; + if (Boolean.TRUE.equals(aWriteIndex)) { + bWriteIndex = randomFrom(Boolean.FALSE, null); + } else { + bWriteIndex = randomFrom(Boolean.TRUE, Boolean.FALSE, null); + } + // when only one index/alias pair exist + MetaData metaData = MetaData.builder().put(buildIndexMetaData(indexA, alias, aWriteIndex)).build(); + + // when alias points to two indices, but valid + // one of the following combinations: [(null, null), (null, true), (null, false), (false, false)] + MetaData.builder(metaData).put(buildIndexMetaData(indexB, alias, bWriteIndex)).build(); + + // when too many write indices + Exception exception = expectThrows(IllegalStateException.class, + () -> { + IndexMetaData.Builder metaA = buildIndexMetaData(indexA, alias, true); + IndexMetaData.Builder metaB = buildIndexMetaData(indexB, alias, true); + MetaData.builder().put(metaA).put(metaB).build(); + }); + assertThat(exception.getMessage(), startsWith("alias [" + alias + "] has more than one write index [")); + } + public void testResolveIndexRouting() { IndexMetaData.Builder builder = IndexMetaData.builder("index") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) @@ -428,6 +456,13 @@ public class MetaDataTests extends ESTestCase { } } + private IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) { + return IndexMetaData.builder(name) + .settings(settings(Version.CURRENT)).creationDate(randomNonNegativeLong()) + .putAlias(AliasMetaData.builder(alias).writeIndex(writeIndex)) + .numberOfShards(1).numberOfReplicas(0); + } + @SuppressWarnings("unchecked") private static void assertIndexMappingsNoFields(ImmutableOpenMap> mappings, String index) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index 3b80d1f6e2c..bde478eb363 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -111,7 +111,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .putMapping("mapping1", MAPPING_SOURCE1) .putMapping("mapping2", MAPPING_SOURCE2) .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) - .putAlias(newAliasMetaDataBuilder("alias2")) + .putAlias(newAliasMetaDataBuilder("alias2").writeIndex(randomBoolean() ? null : randomBoolean())) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) .put(IndexTemplateMetaData.builder("foo") .patterns(Collections.singletonList("bar")) @@ -132,7 +132,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .putMapping("mapping1", MAPPING_SOURCE1) .putMapping("mapping2", MAPPING_SOURCE2) .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) - .putAlias(newAliasMetaDataBuilder("alias2")) + .putAlias(newAliasMetaDataBuilder("alias2").writeIndex(randomBoolean() ? null : randomBoolean())) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) .put(IndexTemplateMetaData.builder("foo") .patterns(Collections.singletonList("bar")) @@ -146,7 +146,6 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .build(); String metaDataSource = MetaData.Builder.toXContent(metaData); -// System.out.println("ToJson: " + metaDataSource); MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource)); @@ -270,6 +269,8 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); assertThat(indexMetaData.getAliases().get("alias2").filter(), nullValue()); + assertThat(indexMetaData.getAliases().get("alias2").writeIndex(), + equalTo(metaData.index("test11").getAliases().get("alias2").writeIndex())); assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4")); assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); @@ -288,6 +289,8 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); assertThat(indexMetaData.getAliases().get("alias2").filter(), nullValue()); + assertThat(indexMetaData.getAliases().get("alias2").writeIndex(), + equalTo(metaData.index("test12").getAliases().get("alias2").writeIndex())); assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4")); assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 06499aa544e..c9ca1637b1a 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1518,9 +1518,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(); assertAcked(client.admin().indices().prepareAliases() - .addAlias("test-idx-1", "alias-1") - .addAlias("test-idx-2", "alias-2") - .addAlias("test-idx-3", "alias-3") + .addAlias("test-idx-1", "alias-1", false) + .addAlias("test-idx-2", "alias-2", false) + .addAlias("test-idx-3", "alias-3", false) ); logger.info("--> indexing some data"); diff --git a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java index 3e42e3b304e..e88a9f0a38d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java @@ -138,6 +138,10 @@ public final class RandomCreateIndexGenerator { alias.filter("{\"term\":{\"year\":2016}}"); } + if (randomBoolean()) { + alias.writeIndex(randomBoolean()); + } + return alias; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java index 991f421265e..b11dfd47651 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java @@ -17,11 +17,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.MlMetaIndex; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.SortedSet; +import java.util.TreeSet; public class MlFilter implements ToXContentObject, Writeable { @@ -53,9 +54,9 @@ public class MlFilter implements ToXContentObject, Writeable { private final String id; private final String description; - private final List items; + private final SortedSet items; - public MlFilter(String id, String description, List items) { + public MlFilter(String id, String description, SortedSet items) { this.id = Objects.requireNonNull(id, ID.getPreferredName() + " must not be null"); this.description = description; this.items = Objects.requireNonNull(items, ITEMS.getPreferredName() + " must not be null"); @@ -68,7 +69,8 @@ public class MlFilter implements ToXContentObject, Writeable { } else { description = null; } - items = Arrays.asList(in.readStringArray()); + items = new TreeSet<>(); + items.addAll(Arrays.asList(in.readStringArray())); } @Override @@ -103,8 +105,8 @@ public class MlFilter implements ToXContentObject, Writeable { return description; } - public List getItems() { - return new ArrayList<>(items); + public SortedSet getItems() { + return Collections.unmodifiableSortedSet(items); } @Override @@ -142,7 +144,7 @@ public class MlFilter implements ToXContentObject, Writeable { private String id; private String description; - private List items = Collections.emptyList(); + private SortedSet items = new TreeSet<>(); private Builder() {} @@ -162,12 +164,13 @@ public class MlFilter implements ToXContentObject, Writeable { } public Builder setItems(List items) { - this.items = items; + this.items = new TreeSet<>(); + this.items.addAll(items); return this; } public Builder setItems(String... items) { - this.items = Arrays.asList(items); + setItems(Arrays.asList(items)); return this; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java index 78d87b82839..9ac6683f004 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java @@ -11,10 +11,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.TreeSet; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -40,7 +39,7 @@ public class MlFilterTests extends AbstractSerializingTestCase { } int size = randomInt(10); - List items = new ArrayList<>(size); + TreeSet items = new TreeSet<>(); for (int i = 0; i < size; i++) { items.add(randomAlphaOfLengthBetween(1, 20)); } @@ -58,7 +57,7 @@ public class MlFilterTests extends AbstractSerializingTestCase { } public void testNullId() { - NullPointerException ex = expectThrows(NullPointerException.class, () -> new MlFilter(null, "", Collections.emptyList())); + NullPointerException ex = expectThrows(NullPointerException.class, () -> new MlFilter(null, "", new TreeSet<>())); assertEquals(MlFilter.ID.getPreferredName() + " must not be null", ex.getMessage()); } @@ -88,4 +87,14 @@ public class MlFilterTests extends AbstractSerializingTestCase { MlFilter.LENIENT_PARSER.apply(parser, null); } } + + public void testItemsAreSorted() { + MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build(); + assertThat(filter.getItems(), contains("a", "b", "c")); + } + + public void testGetItemsReturnsUnmodifiable() { + MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build(); + expectThrows(UnsupportedOperationException.class, () -> filter.getItems().add("x")); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml index a1f7eee0dcc..2b7b86673e0 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml @@ -22,7 +22,7 @@ setup: filter_id: filter-foo body: > { - "items": ["abc", "xyz"] + "items": ["xyz", "abc"] } - do: