diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 50cd572ced6..88331746315 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -109,7 +109,7 @@ By default, internal versioning is used that starts at 1 and increments with each update, deletes included. Optionally, the version number can be supplemented with an external value (for example, if maintained in a database). To enable this functionality, `version_type` should be set to -`external`. The value provided must be a numeric, long value greater than 0, +`external`. The value provided must be a numeric, long value greater or equal to 0, and less than around 9.2e+18. When using the external version type, instead of checking for a matching version number, the system checks to see if the version number passed to the index request is greater than the @@ -138,12 +138,13 @@ of the stored document. `external` or `external_gt`:: only index the document if the given version is strictly higher than the version of the stored document *or* if there is no existing document. The given -version will be used as the new version and will be stored with the new document. +version will be used as the new version and will be stored with the new document. The supplied +version must be a non-negative long number. `external_gte`:: only index the document if the given version is *equal* or higher than the version of the stored document. If there is no existing document the operation will succeed as well. The given version will be used as the new version -and will be stored with the new document. +and will be stored with the new document. The supplied version must be a non-negative long number. `force`:: the document will be indexed regardless of the version of the stored document or if there is no existing document. The given version will be used as the new version and will be stored diff --git a/rest-api-spec/test/index/35_external_version.yaml b/rest-api-spec/test/index/35_external_version.yaml index 0f8f923a07b..f17e6b74931 100644 --- a/rest-api-spec/test/index/35_external_version.yaml +++ b/rest-api-spec/test/index/35_external_version.yaml @@ -1,6 +1,17 @@ --- "External version": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + version_type: external + version: 0 + + - match: { _version: 0 } + - do: index: index: test_1 @@ -10,7 +21,7 @@ version_type: external version: 5 - - match: { _version: 5} + - match: { _version: 5 } - do: catch: conflict @@ -22,6 +33,16 @@ version_type: external version: 5 + - do: + catch: conflict + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + version_type: external + version: 0 + - do: index: index: test_1 diff --git a/rest-api-spec/test/index/36_external_gte_version.yaml b/rest-api-spec/test/index/36_external_gte_version.yaml index 219f03128e9..dccbe02ea14 100644 --- a/rest-api-spec/test/index/36_external_gte_version.yaml +++ b/rest-api-spec/test/index/36_external_gte_version.yaml @@ -1,6 +1,17 @@ --- "External GTE version": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + version_type: external_gte + version: 0 + + - match: { _version: 0} + - do: index: index: test_1 @@ -20,7 +31,7 @@ id: 1 body: { foo: bar } version_type: external_gte - version: 4 + version: 0 - do: index: diff --git a/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 60991cfad71..67022f898b4 100644 --- a/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.support.replication.ShardReplicationOperationReq import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import java.io.IOException; @@ -205,7 +206,7 @@ public class DeleteRequest extends ShardReplicationOperationRequest() { @Override diff --git a/src/main/java/org/elasticsearch/action/delete/index/IndexDeleteRequest.java b/src/main/java/org/elasticsearch/action/delete/index/IndexDeleteRequest.java index c7d4c7a8283..c1b563b6a2b 100644 --- a/src/main/java/org/elasticsearch/action/delete/index/IndexDeleteRequest.java +++ b/src/main/java/org/elasticsearch/action/delete/index/IndexDeleteRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.support.replication.IndexReplicationOperationRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.uid.Versions; import java.io.IOException; @@ -72,7 +73,7 @@ public class IndexDeleteRequest extends IndexReplicationOperationRequest { } this.versionType = VersionType.fromValue(in.readByte()); - this.version = in.readVLong(); + this.version = Versions.readVersionWithVLongForBW(in); fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); } @@ -298,7 +298,7 @@ public class GetRequest extends SingleShardOperationRequest { } out.writeByte(versionType.getValue()); - out.writeVLong(version); + Versions.writeVersionWithVLongForBW(version, out); FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); } diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index 6bce61cba98..5cb64f8c304 100644 --- a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -169,7 +169,7 @@ public class MultiGetRequest extends ActionRequest implements I fields[i] = in.readString(); } } - version = in.readVLong(); + version = Versions.readVersionWithVLongForBW(in); versionType = VersionType.fromValue(in.readByte()); fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); @@ -190,7 +190,7 @@ public class MultiGetRequest extends ActionRequest implements I } } - out.writeVLong(version); + Versions.writeVersionWithVLongForBW(version, out); out.writeByte(versionType.getValue()); FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java index 20d2cbe08dc..454dd051729 100644 --- a/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.single.shard.SingleShardOperationRequest import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.source.FetchSourceContext; @@ -138,7 +139,7 @@ public class MultiGetShardRequest extends SingleShardOperationRequest opType = OpType.fromId(in.readByte()); refresh = in.readBoolean(); - version = in.readLong(); + version = Versions.readVersion(in); versionType = VersionType.fromValue(in.readByte()); if (in.getVersion().onOrAfter(Version.V_1_2_0)) { autoGeneratedId = in.readBoolean(); @@ -651,7 +651,7 @@ public class IndexRequest extends ShardReplicationOperationRequest out.writeBytesReference(source); out.writeByte(opType.id()); out.writeBoolean(refresh); - out.writeLong(version); + Versions.writeVersion(version, out); out.writeByte(versionType.getValue()); if (out.getVersion().onOrAfter(Version.V_1_2_0)) { out.writeBoolean(autoGeneratedId); diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 8972f379473..365bddc393e 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -609,7 +609,7 @@ public class UpdateRequest extends InstanceShardOperationRequest upsertRequest.readFrom(in); } docAsUpsert = in.readBoolean(); - version = in.readLong(); + version = Versions.readVersion(in); versionType = VersionType.fromValue(in.readByte()); } @@ -655,7 +655,7 @@ public class UpdateRequest extends InstanceShardOperationRequest upsertRequest.writeTo(out); } out.writeBoolean(docAsUpsert); - out.writeLong(version); + Versions.writeVersion(version, out); out.writeByte(versionType.getValue()); } diff --git a/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java b/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java index a978d62dc03..f85549cd2b6 100644 --- a/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java +++ b/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java @@ -22,7 +22,10 @@ package org.elasticsearch.common.lucene.uid; import org.apache.lucene.index.*; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.Numbers; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.internal.VersionFieldMapper; @@ -32,11 +35,55 @@ import java.util.List; /** Utility class to resolve the Lucene doc ID and version for a given uid. */ public class Versions { - public static final long MATCH_ANY = 0L; // Version was not specified by the user + public static final long MATCH_ANY = -3L; // Version was not specified by the user + // the value for MATCH_ANY before ES 1.2.0 - will be removed + public static final long MATCH_ANY_PRE_1_2_0 = 0L; public static final long NOT_FOUND = -1L; public static final long NOT_SET = -2L; - private Versions() {} + public static void writeVersion(long version, StreamOutput out) throws IOException { + if (out.getVersion().before(Version.V_1_2_0) && version == MATCH_ANY) { + // we have to send out a value the node will understand + version = MATCH_ANY_PRE_1_2_0; + } + out.writeLong(version); + } + + public static long readVersion(StreamInput in) throws IOException { + long version = in.readLong(); + if (in.getVersion().before(Version.V_1_2_0) && version == MATCH_ANY_PRE_1_2_0) { + version = MATCH_ANY; + } + return version; + } + + public static void writeVersionWithVLongForBW(long version, StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_1_2_0)) { + out.writeLong(version); + return; + } + + if (version == MATCH_ANY) { + // we have to send out a value the node will understand + version = MATCH_ANY_PRE_1_2_0; + } + out.writeVLong(version); + } + + public static long readVersionWithVLongForBW(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_1_2_0)) { + return in.readLong(); + } else { + long version = in.readVLong(); + if (version == MATCH_ANY_PRE_1_2_0) { + return MATCH_ANY; + } + return version; + } + } + + private Versions() { + } /** Wraps an {@link AtomicReaderContext}, a doc ID relative to the context doc base and a version. */ public static class DocIdAndVersion { diff --git a/src/main/java/org/elasticsearch/index/VersionType.java b/src/main/java/org/elasticsearch/index/VersionType.java index 40a91978e46..35a69d9b21a 100644 --- a/src/main/java/org/elasticsearch/index/VersionType.java +++ b/src/main/java/org/elasticsearch/index/VersionType.java @@ -40,7 +40,8 @@ public enum VersionType { if (currentVersion == Versions.NOT_SET) { return false; } - if (expectedVersion == Versions.MATCH_ANY) { + // we need to allow pre 1.2.0 match any as requests can come in for java code where the may be hardcoded + if (expectedVersion == Versions.MATCH_ANY || expectedVersion == Versions.MATCH_ANY_PRE_1_2_0) { return false; } if (currentVersion == Versions.NOT_FOUND) { @@ -60,13 +61,13 @@ public enum VersionType { @Override public boolean validateVersionForWrites(long version) { // not allowing Versions.NOT_FOUND as it is not a valid input value. - return version > 0L || version == Versions.MATCH_ANY; + return version > 0L || version == Versions.MATCH_ANY || version == Versions.MATCH_ANY_PRE_1_2_0; } @Override public boolean validateVersionForReads(long version) { // not allowing Versions.NOT_FOUND as it is not a valid input value. - return version > 0L || version == Versions.MATCH_ANY; + return version > 0L || version == Versions.MATCH_ANY || version == Versions.MATCH_ANY_PRE_1_2_0; } @Override @@ -118,12 +119,12 @@ public enum VersionType { @Override public boolean validateVersionForWrites(long version) { - return version > 0L; + return version >= 0L; } @Override public boolean validateVersionForReads(long version) { - return version > 0L || version == Versions.MATCH_ANY; + return version >= 0L || version == Versions.MATCH_ANY; } }, @@ -169,12 +170,12 @@ public enum VersionType { @Override public boolean validateVersionForWrites(long version) { - return version > 0L; + return version >= 0L; } @Override public boolean validateVersionForReads(long version) { - return version > 0L || version == Versions.MATCH_ANY; + return version >= 0L || version == Versions.MATCH_ANY; } }, @@ -208,12 +209,12 @@ public enum VersionType { @Override public boolean validateVersionForWrites(long version) { - return version > 0L; + return version >= 0L; } @Override public boolean validateVersionForReads(long version) { - return version > 0L || version == Versions.MATCH_ANY; + return version >= 0L || version == Versions.MATCH_ANY; } }; diff --git a/src/main/java/org/elasticsearch/index/translog/Translog.java b/src/main/java/org/elasticsearch/index/translog/Translog.java index 3771558e281..5a86ec09bfd 100644 --- a/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -351,7 +351,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { } } if (version >= 3) { - this.version = in.readLong(); + this.version = Versions.readVersion(in); } if (version >= 4) { this.timestamp = in.readLong(); @@ -384,7 +384,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { out.writeBoolean(true); out.writeString(parent); } - out.writeLong(version); + Versions.writeVersion(version, out); out.writeLong(timestamp); out.writeLong(ttl); out.writeByte(versionType.getValue()); @@ -494,7 +494,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { } } if (version >= 3) { - this.version = in.readLong(); + this.version = Versions.readVersion(in); } if (version >= 4) { this.timestamp = in.readLong(); @@ -527,7 +527,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { out.writeBoolean(true); out.writeString(parent); } - out.writeLong(version); + Versions.writeVersion(version, out); out.writeLong(timestamp); out.writeLong(ttl); out.writeByte(versionType.getValue()); @@ -586,7 +586,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { int version = in.readVInt(); // version uid = new Term(in.readString(), in.readString()); if (version >= 1) { - this.version = in.readLong(); + this.version = Versions.readVersion(in); } if (version >= 2) { this.versionType = VersionType.fromValue(in.readByte()); @@ -600,7 +600,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent { out.writeVInt(SERIALIZATION_FORMAT); out.writeString(uid.field()); out.writeString(uid.text()); - out.writeLong(version); + Versions.writeVersion(version, out); out.writeByte(versionType.getValue()); } } diff --git a/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/src/main/java/org/elasticsearch/rest/action/support/RestActions.java index 8cc900284eb..3d7e4853492 100644 --- a/src/main/java/org/elasticsearch/rest/action/support/RestActions.java +++ b/src/main/java/org/elasticsearch/rest/action/support/RestActions.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.query.QueryBuilders; @@ -40,13 +41,13 @@ public class RestActions { public static long parseVersion(RestRequest request) { if (request.hasParam("version")) { - return request.paramAsLong("version", 0); + return request.paramAsLong("version", Versions.MATCH_ANY); } String ifMatch = request.header("If-Match"); if (ifMatch != null) { return Long.parseLong(ifMatch); } - return 0; + return Versions.MATCH_ANY; } static final class Fields { diff --git a/src/test/java/org/elasticsearch/index/VersionTypeTests.java b/src/test/java/org/elasticsearch/index/VersionTypeTests.java index f539f9726f8..d13e8010d38 100644 --- a/src/test/java/org/elasticsearch/index/VersionTypeTests.java +++ b/src/test/java/org/elasticsearch/index/VersionTypeTests.java @@ -30,19 +30,31 @@ public class VersionTypeTests extends ElasticsearchTestCase { public void testInternalVersionConflict() throws Exception { assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(10, Versions.MATCH_ANY)); // if we don't have a version in the index we accept everything assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_SET, 10)); assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, Versions.MATCH_ANY)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_SET, Versions.MATCH_ANY)); // if we didn't find a version (but the index does support it), we don't like it unless MATCH_ANY - assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND)); assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10)); + assertTrue(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_FOUND, 10)); assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.MATCH_ANY)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_FOUND, Versions.MATCH_ANY)); + + // test 0 is still matching any for backwards compatibility with versions <1.2.0 + assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY_PRE_1_2_0)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(10, Versions.MATCH_ANY_PRE_1_2_0)); + // and the stupid usual case assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, 10)); + assertFalse(VersionType.INTERNAL.isVersionConflictForReads(10, 10)); assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(9, 10)); + assertTrue(VersionType.INTERNAL.isVersionConflictForReads(9, 10)); assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(10, 9)); + assertTrue(VersionType.INTERNAL.isVersionConflictForReads(10, 9)); // Old indexing code, dictating behavior // if (expectedVersion != Versions.MATCH_ANY && currentVersion != Versions.NOT_SET) {