Version types EXTERNAL & EXTERNAL_GTE test for version equality in read operation & disallow them in the Update API

Separate version check logic for reads and writes for all version types, which allows different behavior in these cases.
Change `VersionType.EXTERNAL` & `VersionType.EXTERNAL_GTE` to behave the same as `VersionType.INTERNAL` for read operations.
The previous behavior was fit for writes but is useless in reads.

This commit also makes the usage of `EXTERNAL` & `EXTERNAL_GTE` in the update api raise a validation error as it make cause data to
be lost.

Closes #5663 , Closes #5661, Closes #5929
This commit is contained in:
Boaz Leskes 2014-04-23 11:13:25 +02:00
parent 080c4ade25
commit 051beb51a3
22 changed files with 458 additions and 221 deletions

View File

@ -212,3 +212,15 @@ redirected to one of the replicas within that shard id and returns the
result. The replicas are the primary shard and its replicas within that
shard id group. This means that the more replicas we will have, the
better GET scaling we will have.
[float]
[[get-versioning]
=== Versioning support
You can use the `version` parameter to retrieve the document only if
it's current version is equal to the specified one. This behavior is the same
for all version types with the exception of version type `FORCE` which always
retrieves the document.
Note that Elasticsearch do not store older versions of documents. Only the current version can be retrieved.

View File

@ -165,6 +165,14 @@ including:
Support `_source` to return the full updated
source.
`version` & `version_type`:: the Update API uses the Elasticsearch's versioning
support internally to make sure the document doesn't change
during the update. You can use the `version` parameter to specify that the
document should only be updated if it's version matches the one specified.
By setting version type to `force` you can force the new version of the document
after update (use with care! with `force` there is no guaranty the document
didn't change).Version types `external` & `external_gte` are not supported.
And also support `retry_on_conflict` which controls how many times to
retry if there is a version conflict between getting the document and

View File

@ -7,85 +7,85 @@
"paths": ["/{index}/{type}/{id}/_update"],
"parts": {
"id": {
"type" : "string",
"required" : true,
"description" : "Document ID"
"type": "string",
"required": true,
"description": "Document ID"
},
"index": {
"type" : "string",
"required" : true,
"description" : "The name of the index"
"type": "string",
"required": true,
"description": "The name of the index"
},
"type": {
"type" : "string",
"required" : true,
"description" : "The type of the document"
"type": "string",
"required": true,
"description": "The type of the document"
}
},
"params": {
"consistency": {
"type" : "enum",
"options" : ["one", "quorum", "all"],
"description" : "Explicit write consistency setting for the operation"
"type": "enum",
"options": ["one", "quorum", "all"],
"description": "Explicit write consistency setting for the operation"
},
"fields": {
"type": "list",
"description" : "A comma-separated list of fields to return in the response"
"description": "A comma-separated list of fields to return in the response"
},
"lang": {
"type" : "string",
"description" : "The script language (default: mvel)"
"type": "string",
"description": "The script language (default: mvel)"
},
"parent": {
"type" : "string",
"description" : "ID of the parent document"
"type": "string",
"description": "ID of the parent document"
},
"refresh": {
"type" : "boolean",
"description" : "Refresh the index after performing the operation"
"type": "boolean",
"description": "Refresh the index after performing the operation"
},
"replication": {
"type" : "enum",
"options" : ["sync","async"],
"default" : "sync",
"description" : "Specific replication type"
"type": "enum",
"options": ["sync", "async"],
"default": "sync",
"description": "Specific replication type"
},
"retry_on_conflict": {
"type" : "number",
"description" : "Specify how many times should the operation be retried when a conflict occurs (default: 0)"
"type": "number",
"description": "Specify how many times should the operation be retried when a conflict occurs (default: 0)"
},
"routing": {
"type" : "string",
"description" : "Specific routing value"
"type": "string",
"description": "Specific routing value"
},
"script": {
"description" : "The URL-encoded script definition (instead of using request body)"
"description": "The URL-encoded script definition (instead of using request body)"
},
"timeout": {
"type" : "time",
"description" : "Explicit operation timeout"
"type": "time",
"description": "Explicit operation timeout"
},
"timestamp": {
"type" : "time",
"description" : "Explicit timestamp for the document"
"type": "time",
"description": "Explicit timestamp for the document"
},
"ttl": {
"type" : "duration",
"description" : "Expiration time for the document"
"type": "duration",
"description": "Expiration time for the document"
},
"version" : {
"type" : "number",
"description" : "Explicit version number for concurrency control"
"version": {
"type": "number",
"description": "Explicit version number for concurrency control"
},
"version_type": {
"type" : "enum",
"options" : ["internal", "external", "external_gte", "force"],
"description" : "Specific version type"
"type": "enum",
"options": ["internal", "force"],
"description": "Specific version type"
}
}
},
"body": {
"description" : "The request definition using either `script` or partial `doc`"
"description": "The request definition using either `script` or partial `doc`"
}
}
}

View File

@ -0,0 +1,115 @@
---
"Versions":
- do:
index:
index: test_1
type: test
id: 1
body: { foo: bar }
- match: { _version: 1}
- do:
index:
index: test_1
type: test
id: 1
body: { foo: bar }
- match: { _version: 2}
- do:
get:
index: test_1
type: test
id: 1
version: 2
- match: { _id: "1" }
- do:
catch: conflict
get:
index: test_1
type: test
id: 1
version: 1
- do:
get:
index: test_1
type: test
id: 1
version: 2
version_type: external
- match: { _id: "1" }
- do:
catch: conflict
get:
index: test_1
type: test
id: 1
version: 10
version_type: external
- do:
catch: conflict
get:
index: test_1
type: test
id: 1
version: 1
version_type: external
- do:
get:
index: test_1
type: test
id: 1
version: 2
version_type: external_gte
- match: { _id: "1" }
- do:
catch: conflict
get:
index: test_1
type: test
id: 1
version: 10
version_type: external_gte
- do:
catch: conflict
get:
index: test_1
type: test
id: 1
version: 1
version_type: external_gte
- do:
get:
index: test_1
type: test
id: 1
version: 2
version_type: force
- match: { _id: "1" }
- do:
get:
index: test_1
type: test
id: 1
version: 10
version_type: force
- match: { _id: "1" }
- do:
get:
index: test_1
type: test
id: 1
version: 1
version_type: force
- match: { _id: "1" }

View File

@ -12,29 +12,8 @@
doc: { foo: baz }
upsert: { foo: bar }
- do:
update:
index: test_1
type: test
id: 1
body:
doc: { foo: baz }
upsert: { foo: bar }
- match: { _version: 1}
- do:
catch: conflict
update:
index: test_1
type: test
id: 1
version: 2
body:
doc: { foo: baz }
upsert: { foo: bar }
- do:
update:
index: test_1
type: test
@ -43,5 +22,3 @@
body:
doc: { foo: baz }
upsert: { foo: bar }
- match: { _version: 2}

View File

@ -1,21 +1,8 @@
---
"External version":
"Not supported versions":
- do:
update:
index: test_1
type: test
id: 1
version: 2
version_type: external
body:
doc: { foo: baz }
upsert: { foo: bar }
- match: { _version: 2 }
- do:
catch: conflict
catch: /Validation/
update:
index: test_1
type: test
@ -27,14 +14,14 @@
upsert: { foo: bar }
- do:
catch: /Validation/
update:
index: test_1
type: test
id: 1
version: 3
version_type: external
version: 2
version_type: external_gte
body:
doc: { foo: baz }
upsert: { foo: bar }
- match: { _version: 3 }

View File

@ -436,7 +436,7 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation
throw new WriteFailure(t, mappingsToUpdate);
}
assert indexRequest.versionType().validateVersion(indexRequest.version());
assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());
IndexResponse indexResponse = new IndexResponse(indexRequest.index(), indexRequest.type(), indexRequest.id(), version, created);
@ -450,7 +450,7 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation
deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
deleteRequest.version(delete.version());
assert deleteRequest.versionType().validateVersion(deleteRequest.version());
assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
DeleteResponse deleteResponse = new DeleteResponse(deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), delete.version(), delete.found());
return new WriteResult(deleteResponse, null, null);

View File

@ -94,8 +94,8 @@ public class DeleteRequest extends ShardReplicationOperationRequest<DeleteReques
if (id == null) {
validationException = addValidationError("id is missing", validationException);
}
if (!versionType.validateVersion(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type ["+ versionType.name() + "]", validationException);
if (!versionType.validateVersionForWrites(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
}
return validationException;
}

View File

@ -191,7 +191,7 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct
request.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
request.version(delete.version());
assert request.versionType().validateVersion(request.version());
assert request.versionType().validateVersionForWrites(request.version());
if (request.refresh()) {
try {

View File

@ -120,7 +120,7 @@ public class TransportShardDeleteAction extends TransportShardReplicationOperati
// IndexDeleteAction doesn't support version type at the moment. Hard coded for the INTERNAL version
delete = new Engine.Delete(delete, VersionType.INTERNAL.versionTypeForReplicationAndRecovery());
assert delete.versionType().validateVersion(delete.version());
assert delete.versionType().validateVersionForWrites(delete.version());
indexShard.delete(delete);

View File

@ -95,6 +95,10 @@ public class GetRequest extends SingleShardOperationRequest<GetRequest> {
if (id == null) {
validationException = ValidateActions.addValidationError("id is missing", validationException);
}
if (!versionType.validateVersionForReads(version)) {
validationException = ValidateActions.addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]",
validationException);
}
return validationException;
}

View File

@ -62,7 +62,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
* @see org.elasticsearch.client.Client#index(IndexRequest)
*/
public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest> {
/**
* Operation type controls if the type of the index operation.
*/
@ -175,8 +175,8 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
if (source == null) {
validationException = addValidationError("source is missing", validationException);
}
if (!versionType.validateVersion(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type ["+ versionType.name() + "]", validationException);
if (!versionType.validateVersionForWrites(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
}
return validationException;
}

View File

@ -222,7 +222,7 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi
request.version(version);
request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
assert request.versionType().validateVersion(request.version());
assert request.versionType().validateVersionForWrites(request.version());
IndexResponse response = new IndexResponse(request.index(), request.type(), request.id(), version, created);
return new PrimaryResponse<>(shardRequest.request, response, op);

View File

@ -106,7 +106,9 @@ public class UpdateHelper extends AbstractComponent {
}
long updateVersion = getResult.getVersion();
if (request.versionType() != VersionType.INTERNAL) {
assert request.versionType() == VersionType.FORCE;
updateVersion = request.version(); // remember, match_any is excluded by the conflict test
}

View File

@ -96,12 +96,17 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
validationException = addValidationError("id is missing", validationException);
}
if (version != Versions.MATCH_ANY && retryOnConflict > 0) {
validationException = addValidationError("can't provide both retry_on_conflict and a specific version", validationException);
}
if (!(versionType == VersionType.INTERNAL || versionType == VersionType.FORCE)) {
validationException = addValidationError("version type [" + versionType + "] is not supported by the update API", validationException);
} else {
if (!versionType.validateVersion(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type ["+ versionType.name() + "]", validationException);
if (version != Versions.MATCH_ANY && retryOnConflict > 0) {
validationException = addValidationError("can't provide both retry_on_conflict and a specific version", validationException);
}
if (!versionType.validateVersionForWrites(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
}
}
if (script == null && doc == null) {

View File

@ -27,7 +27,16 @@ import org.elasticsearch.common.lucene.uid.Versions;
public enum VersionType {
INTERNAL((byte) 0) {
@Override
public boolean isVersionConflict(long currentVersion, long expectedVersion) {
public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
return isVersionConflict(currentVersion, expectedVersion);
}
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
return isVersionConflict(currentVersion, expectedVersion);
}
private boolean isVersionConflict(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@ -49,7 +58,13 @@ public enum VersionType {
}
@Override
public boolean validateVersion(long version) {
public boolean validateVersionForWrites(long version) {
// not allowing Versions.NOT_FOUND as it is not a valid input value.
return version > 0L || version == Versions.MATCH_ANY;
}
@Override
public boolean validateVersionForReads(long version) {
// not allowing Versions.NOT_FOUND as it is not a valid input value.
return version > 0L || version == Versions.MATCH_ANY;
}
@ -63,7 +78,7 @@ public enum VersionType {
},
EXTERNAL((byte) 1) {
@Override
public boolean isVersionConflict(long currentVersion, long expectedVersion) {
public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@ -79,24 +94,42 @@ public enum VersionType {
return false;
}
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
if (expectedVersion == Versions.MATCH_ANY) {
return false;
}
if (currentVersion == Versions.NOT_FOUND) {
return true;
}
if (currentVersion != expectedVersion) {
return true;
}
return false;
}
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
}
@Override
public boolean validateVersion(long version) {
public boolean validateVersionForWrites(long version) {
return version > 0L;
}
@Override
public boolean validateVersionForReads(long version) {
return version > 0L || version == Versions.MATCH_ANY;
}
},
EXTERNAL_GTE((byte) 2) {
/**
* - always returns false if currentVersion == {@link Versions#NOT_SET}
* - always conflict if expectedVersion == {@link Versions#MATCH_ANY} (we need something to set)
* - accepts currentVersion == {@link Versions#NOT_FOUND}
*/
@Override
public boolean isVersionConflict(long currentVersion, long expectedVersion) {
public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@ -112,27 +145,45 @@ public enum VersionType {
return false;
}
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
if (expectedVersion == Versions.MATCH_ANY) {
return false;
}
if (currentVersion == Versions.NOT_FOUND) {
return true;
}
if (currentVersion != expectedVersion) {
return true;
}
return false;
}
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
}
@Override
public boolean validateVersion(long version) {
public boolean validateVersionForWrites(long version) {
return version > 0L;
}
@Override
public boolean validateVersionForReads(long version) {
return version > 0L || version == Versions.MATCH_ANY;
}
},
/**
* Warning: this version type should be used with care. Concurrent indexing may result in loss of data on replicas
*/
FORCE((byte) 3) {
/**
* - always returns false if currentVersion == {@link Versions#NOT_SET}
* - always conflict if expectedVersion == {@link Versions#MATCH_ANY} (we need something to set)
* - accepts currentVersion == {@link Versions#NOT_FOUND}
*/
@Override
public boolean isVersionConflict(long currentVersion, long expectedVersion) {
public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@ -145,15 +196,26 @@ public enum VersionType {
return false;
}
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
return false;
}
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
}
@Override
public boolean validateVersion(long version) {
public boolean validateVersionForWrites(long version) {
return version > 0L;
}
@Override
public boolean validateVersionForReads(long version) {
return version > 0L || version == Versions.MATCH_ANY;
}
};
private final byte value;
@ -171,7 +233,14 @@ public enum VersionType {
*
* @return true if versions conflict false o.w.
*/
public abstract boolean isVersionConflict(long currentVersion, long expectedVersion);
public abstract boolean isVersionConflictForWrites(long currentVersion, long expectedVersion);
/**
* Checks whether the current version conflicts with the expected version, based on the current version type.
*
* @return true if versions conflict false o.w.
*/
public abstract boolean isVersionConflictForReads(long currentVersion, long expectedVersion);
/**
* Returns the new version for a document, based on its current one and the specified in the request
@ -180,12 +249,22 @@ public enum VersionType {
*/
public abstract long updateVersion(long currentVersion, long expectedVersion);
/** validate the version is a valid value for this type.
/**
* validate the version is a valid value for this type when writing.
*
* @return true if valid, false o.w
*/
public abstract boolean validateVersion(long version);
public abstract boolean validateVersionForWrites(long version);
/** Some version types require different semantics for primary and replicas. This version allows
/**
* validate the version is a valid value for this type when reading.
*
* @return true if valid, false o.w
*/
public abstract boolean validateVersionForReads(long version);
/**
* Some version types require different semantics for primary and replicas. This version allows
* the type to override the default behavior.
*/
public VersionType versionTypeForReplicationAndRecovery() {

View File

@ -724,8 +724,8 @@ public interface Engine extends IndexShardComponent, CloseableComponent {
private final boolean realtime;
private final Term uid;
private boolean loadSource = true;
private long version;
private VersionType versionType;
private long version = Versions.MATCH_ANY;
private VersionType versionType = VersionType.INTERNAL;
public Get(boolean realtime, Term uid) {
this.realtime = realtime;

View File

@ -319,11 +319,9 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
if (versionValue.delete()) {
return GetResult.NOT_EXISTS;
}
if (get.version() != Versions.MATCH_ANY) {
if (get.versionType().isVersionConflict(versionValue.version(), get.version())) {
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), versionValue.version(), get.version());
}
if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) {
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), versionValue.version(), get.version());
}
if (!get.loadSource()) {
return new GetResult(true, versionValue.version(), null);
@ -351,8 +349,8 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
throw new EngineException(shardId(), "Couldn't resolve version", e);
}
if (get.version() != Versions.MATCH_ANY && docIdAndVersion != null) {
if (get.versionType().isVersionConflict(docIdAndVersion.version, get.version())) {
if (docIdAndVersion != null) {
if (get.versionType().isVersionConflictForReads(docIdAndVersion.version, get.version())) {
Releasables.close(searcher);
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), docIdAndVersion.version, get.version());
@ -416,7 +414,7 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
// same logic as index
long updatedVersion;
long expectedVersion = create.version();
if (create.versionType().isVersionConflict(currentVersion, expectedVersion)) {
if (create.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
if (create.origin() == Operation.Origin.RECOVERY) {
return;
} else {
@ -493,7 +491,7 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
long updatedVersion;
long expectedVersion = index.version();
if (index.versionType().isVersionConflict(currentVersion, expectedVersion)) {
if (index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
if (index.origin() == Operation.Origin.RECOVERY) {
return;
} else {
@ -564,7 +562,7 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
long updatedVersion;
long expectedVersion = delete.version();
if (delete.versionType().isVersionConflict(currentVersion, expectedVersion)) {
if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
if (delete.origin() == Operation.Origin.RECOVERY) {
return;
} else {

View File

@ -363,7 +363,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent {
this.versionType = VersionType.fromValue(in.readByte());
}
assert versionType.validateVersion(version);
assert versionType.validateVersionForWrites(version);
}
@Override
@ -506,7 +506,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent {
this.versionType = VersionType.fromValue(in.readByte());
}
assert versionType.validateVersion(version);
assert versionType.validateVersionForWrites(version);
}
@Override
@ -591,7 +591,7 @@ public interface Translog extends IndexShardComponent, CloseableIndexComponent {
if (version >= 2) {
this.versionType = VersionType.fromValue(in.readByte());
}
assert versionType.validateVersion(version);
assert versionType.validateVersionForWrites(version);
}

View File

@ -41,7 +41,8 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@ -165,13 +166,13 @@ public class BulkTests extends ElasticsearchIntegrationTest {
assertThat(((IndexResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(12l));
bulkResponse = client().prepareBulk()
.add(client().prepareUpdate("test", "type", "e1").setVersion(4l).setDoc("field", "2").setVersion(10).setVersionType(VersionType.EXTERNAL))
.add(client().prepareUpdate("test", "type", "e2").setDoc("field", "2").setVersion(15).setVersionType(VersionType.EXTERNAL))
.add(client().prepareUpdate("test", "type", "e1").setVersion(2l).setDoc("field", "3").setVersion(15).setVersionType(VersionType.EXTERNAL)).get();
.add(client().prepareUpdate("test", "type", "e1").setDoc("field", "2").setVersion(10)) // INTERNAL
.add(client().prepareUpdate("test", "type", "e1").setDoc("field", "3").setVersion(20).setVersionType(VersionType.FORCE))
.add(client().prepareUpdate("test", "type", "e1").setDoc("field", "3").setVersion(20).setVersionType(VersionType.INTERNAL)).get();
assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("Version"));
assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(15l));
assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(15l));
assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(20l));
assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(21l));
}
@Test
@ -358,7 +359,8 @@ public class BulkTests extends ElasticsearchIntegrationTest {
assertAcked(prepareCreate("test").setSettings(
ImmutableSettings.builder()
.put(indexSettings())
.put("index.number_of_replicas", replica)));
.put("index.number_of_replicas", replica)
));
int numDocs = scaledRandomIntBetween(100, 5000);
int bulk = scaledRandomIntBetween(1, 99);
@ -456,13 +458,13 @@ public class BulkTests extends ElasticsearchIntegrationTest {
@Test
public void testFailingVersionedUpdatedOnBulk() throws Exception {
createIndex("test");
index("test","type","1","field","1");
index("test", "type", "1", "field", "1");
final BulkResponse[] responses = new BulkResponse[30];
final CyclicBarrier cyclicBarrier = new CyclicBarrier(responses.length);
Thread[] threads = new Thread[responses.length];
for (int i=0;i<responses.length;i++) {
for (int i = 0; i < responses.length; i++) {
final int threadID = i;
threads[threadID] = new Thread(new Runnable() {
@Override
@ -474,7 +476,7 @@ public class BulkTests extends ElasticsearchIntegrationTest {
}
BulkRequestBuilder requestBuilder = client().prepareBulk();
requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID));
responses[threadID]=requestBuilder.get();
responses[threadID] = requestBuilder.get();
}
});
@ -482,13 +484,15 @@ public class BulkTests extends ElasticsearchIntegrationTest {
}
for (int i=0;i < threads.length; i++) {
for (int i = 0; i < threads.length; i++) {
threads[i].join();
}
int successes = 0;
for (BulkResponse response : responses) {
if (!response.hasFailures()) successes ++;
if (!response.hasFailures()) {
successes++;
}
}
assertThat(successes, equalTo(1));
@ -497,13 +501,13 @@ public class BulkTests extends ElasticsearchIntegrationTest {
@Test // issue 4745
public void preParsingSourceDueToMappingShouldNotBreakCompleteBulkRequest() throws Exception {
XContentBuilder builder = jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("path", "last_modified")
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("path", "last_modified")
.endObject()
.endObject()
.endObject()
.endObject();
.endObject();
assertAcked(prepareCreate("test").addMapping("type", builder));
String brokenBuildRequestData = "{\"index\": {\"_id\": \"1\"}}\n" +
@ -522,13 +526,13 @@ public class BulkTests extends ElasticsearchIntegrationTest {
@Test // issue 4745
public void preParsingSourceDueToRoutingShouldNotBreakCompleteBulkRequest() throws Exception {
XContentBuilder builder = jsonBuilder().startObject()
.startObject("type")
.startObject("_routing")
.field("required", true)
.field("path", "my_routing")
.startObject("type")
.startObject("_routing")
.field("required", true)
.field("path", "my_routing")
.endObject()
.endObject()
.endObject()
.endObject();
.endObject();
assertAcked(prepareCreate("test").addMapping("type", builder));
ensureYellow("test");
@ -549,12 +553,12 @@ public class BulkTests extends ElasticsearchIntegrationTest {
@Test // issue 4745
public void preParsingSourceDueToIdShouldNotBreakCompleteBulkRequest() throws Exception {
XContentBuilder builder = jsonBuilder().startObject()
.startObject("type")
.startObject("_id")
.field("path", "my_id")
.startObject("type")
.startObject("_id")
.field("path", "my_id")
.endObject()
.endObject()
.endObject()
.endObject();
.endObject();
assertAcked(prepareCreate("test").addMapping("type", builder));
ensureYellow("test");
@ -578,7 +582,8 @@ public class BulkTests extends ElasticsearchIntegrationTest {
final CountDownLatch latch = new CountDownLatch(1);
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {}
public void beforeBulk(long executionId, BulkRequest request) {
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
@ -595,7 +600,7 @@ public class BulkTests extends ElasticsearchIntegrationTest {
try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setBulkActions(5)
.setConcurrentRequests(1).setName("foo").build()) {
.setConcurrentRequests(1).setName("foo").build()) {
Map<String, Object> data = Maps.newHashMap();
data.put("foo", "bar");
@ -639,7 +644,8 @@ public class BulkTests extends ElasticsearchIntegrationTest {
final CountDownLatch latch = new CountDownLatch(1);
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {}
public void beforeBulk(long executionId, BulkRequest request) {
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
@ -655,7 +661,7 @@ public class BulkTests extends ElasticsearchIntegrationTest {
};
try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setBulkActions(6)
.setConcurrentRequests(1).setName("foo").build()) {
.setConcurrentRequests(1).setName("foo").build()) {
Map<String, Object> data = Maps.newHashMap();
data.put("foo", "bar");

View File

@ -29,20 +29,20 @@ public class VersionTypeTests extends ElasticsearchTestCase {
@Test
public void testInternalVersionConflict() throws Exception {
assertFalse(VersionType.INTERNAL.isVersionConflict(10, Versions.MATCH_ANY));
assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY));
// if we don't have a version in the index we accept everything
assertFalse(VersionType.INTERNAL.isVersionConflict(Versions.NOT_SET, 10));
assertFalse(VersionType.INTERNAL.isVersionConflict(Versions.NOT_SET, Versions.MATCH_ANY));
assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10));
assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, Versions.MATCH_ANY));
// if we didn't find a version (but the index does support it), we don't like it unless MATCH_ANY
assertTrue(VersionType.INTERNAL.isVersionConflict(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertTrue(VersionType.INTERNAL.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.INTERNAL.isVersionConflict(Versions.NOT_FOUND, Versions.MATCH_ANY));
assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.MATCH_ANY));
// and the stupid usual case
assertFalse(VersionType.INTERNAL.isVersionConflict(10, 10));
assertTrue(VersionType.INTERNAL.isVersionConflict(9, 10));
assertTrue(VersionType.INTERNAL.isVersionConflict(10, 9));
assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, 10));
assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(9, 10));
assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(10, 9));
// Old indexing code, dictating behavior
// if (expectedVersion != Versions.MATCH_ANY && currentVersion != Versions.NOT_SET) {
@ -60,40 +60,60 @@ public class VersionTypeTests extends ElasticsearchTestCase {
@Test
public void testVersionValidation() {
assertTrue(VersionType.EXTERNAL.validateVersion(randomIntBetween(1,Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL.validateVersion(0)); // MATCH_ANY
assertFalse(VersionType.EXTERNAL.validateVersion(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.EXTERNAL.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL.validateVersionForWrites(Versions.MATCH_ANY));
assertFalse(VersionType.EXTERNAL.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.EXTERNAL.validateVersionForReads(Versions.MATCH_ANY));
assertTrue(VersionType.EXTERNAL.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
assertTrue(VersionType.EXTERNAL_GTE.validateVersion(randomIntBetween(1,Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL_GTE.validateVersion(0)); // MATCH_ANY
assertFalse(VersionType.EXTERNAL_GTE.validateVersion(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.EXTERNAL_GTE.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL_GTE.validateVersionForWrites(Versions.MATCH_ANY));
assertFalse(VersionType.EXTERNAL_GTE.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.EXTERNAL_GTE.validateVersionForReads(Versions.MATCH_ANY));
assertTrue(VersionType.EXTERNAL_GTE.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.EXTERNAL_GTE.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
assertTrue(VersionType.FORCE.validateVersion(randomIntBetween(1,Integer.MAX_VALUE)));
assertFalse(VersionType.FORCE.validateVersion(0)); // MATCH_ANY
assertFalse(VersionType.FORCE.validateVersion(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.FORCE.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.FORCE.validateVersionForWrites(Versions.MATCH_ANY));
assertFalse(VersionType.FORCE.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.FORCE.validateVersionForReads(Versions.MATCH_ANY));
assertTrue(VersionType.FORCE.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.FORCE.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
assertTrue(VersionType.INTERNAL.validateVersion(randomIntBetween(1,Integer.MAX_VALUE)));
assertTrue(VersionType.INTERNAL.validateVersion(0)); // MATCH_ANY
assertFalse(VersionType.INTERNAL.validateVersion(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.INTERNAL.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE)));
assertTrue(VersionType.INTERNAL.validateVersionForWrites(Versions.MATCH_ANY));
assertFalse(VersionType.INTERNAL.validateVersionForWrites(randomIntBetween(Integer.MIN_VALUE, 0)));
assertTrue(VersionType.INTERNAL.validateVersionForReads(Versions.MATCH_ANY));
assertTrue(VersionType.INTERNAL.validateVersionForReads(randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(VersionType.INTERNAL.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1)));
}
@Test
public void testExternalVersionConflict() throws Exception {
assertFalse(VersionType.EXTERNAL.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflict(Versions.NOT_SET, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10));
// MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value
assertTrue(VersionType.EXTERNAL.isVersionConflict(10, Versions.MATCH_ANY));
assertTrue(VersionType.EXTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY));
// if we didn't find a version (but the index does support it), we always accept
assertFalse(VersionType.EXTERNAL.isVersionConflict(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.EXTERNAL.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflict(Versions.NOT_FOUND, Versions.MATCH_ANY));
assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertTrue(VersionType.EXTERNAL.isVersionConflictForReads(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertTrue(VersionType.EXTERNAL.isVersionConflictForReads(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflictForReads(Versions.NOT_FOUND, Versions.MATCH_ANY));
// and the standard behavior
assertTrue(VersionType.EXTERNAL.isVersionConflict(10, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflict(9, 10));
assertTrue(VersionType.EXTERNAL.isVersionConflict(10, 9));
assertTrue(VersionType.EXTERNAL.isVersionConflictForWrites(10, 10));
assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(9, 10));
assertTrue(VersionType.EXTERNAL.isVersionConflictForWrites(10, 9));
assertFalse(VersionType.EXTERNAL.isVersionConflictForReads(10, 10));
assertTrue(VersionType.EXTERNAL.isVersionConflictForReads(9, 10));
assertTrue(VersionType.EXTERNAL.isVersionConflictForReads(10, 9));
assertFalse(VersionType.EXTERNAL.isVersionConflictForReads(10, Versions.MATCH_ANY));
// Old indexing code, dictating behavior
@ -110,39 +130,57 @@ public class VersionTypeTests extends ElasticsearchTestCase {
@Test
public void testExternalGTEVersionConflict() throws Exception {
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(Versions.NOT_SET, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_SET, 10));
// MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflict(10, Versions.MATCH_ANY));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(10, Versions.MATCH_ANY));
// if we didn't find a version (but the index does support it), we always accept
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(Versions.NOT_FOUND, Versions.MATCH_ANY));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForReads(Versions.NOT_FOUND, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.MATCH_ANY));
// and the standard behavior
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(10, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflict(9, 10));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflict(10, 9));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(10, 10));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(9, 10));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(10, 9));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForReads(10, 10));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForReads(9, 10));
assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForReads(10, 9));
assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForReads(10, Versions.MATCH_ANY));
}
@Test
public void testForceVersionConflict() throws Exception {
assertFalse(VersionType.FORCE.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.FORCE.isVersionConflict(Versions.NOT_SET, 10));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_SET, 10));
// MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value
assertTrue(VersionType.FORCE.isVersionConflict(10, Versions.MATCH_ANY));
assertTrue(VersionType.FORCE.isVersionConflictForWrites(10, Versions.MATCH_ANY));
// if we didn't find a version (but the index does support it), we always accept
assertFalse(VersionType.FORCE.isVersionConflict(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.FORCE.isVersionConflict(Versions.NOT_FOUND, 10));
assertFalse(VersionType.FORCE.isVersionConflict(Versions.NOT_FOUND, Versions.MATCH_ANY));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10));
assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.NOT_FOUND));
assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, 10));
assertFalse(VersionType.FORCE.isVersionConflictForReads(Versions.NOT_FOUND, Versions.MATCH_ANY));
// and the standard behavior
assertFalse(VersionType.FORCE.isVersionConflict(10, 10));
assertFalse(VersionType.FORCE.isVersionConflict(9, 10));
assertFalse(VersionType.FORCE.isVersionConflict(10, 9));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(10, 10));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(9, 10));
assertFalse(VersionType.FORCE.isVersionConflictForWrites(10, 9));
assertFalse(VersionType.FORCE.isVersionConflictForReads(10, 10));
assertFalse(VersionType.FORCE.isVersionConflictForReads(9, 10));
assertFalse(VersionType.FORCE.isVersionConflictForReads(10, 9));
assertFalse(VersionType.FORCE.isVersionConflictForReads(10, Versions.MATCH_ANY));
}
@Test
@ -158,6 +196,15 @@ public class VersionTypeTests extends ElasticsearchTestCase {
assertThat(VersionType.EXTERNAL.updateVersion(Versions.NOT_FOUND, 10), equalTo(10l));
assertThat(VersionType.EXTERNAL.updateVersion(1, 10), equalTo(10l));
assertThat(VersionType.EXTERNAL_GTE.updateVersion(Versions.NOT_SET, 10), equalTo(10l));
assertThat(VersionType.EXTERNAL_GTE.updateVersion(Versions.NOT_FOUND, 10), equalTo(10l));
assertThat(VersionType.EXTERNAL_GTE.updateVersion(1, 10), equalTo(10l));
assertThat(VersionType.EXTERNAL_GTE.updateVersion(10, 10), equalTo(10l));
assertThat(VersionType.FORCE.updateVersion(Versions.NOT_SET, 10), equalTo(10l));
assertThat(VersionType.FORCE.updateVersion(Versions.NOT_FOUND, 10), equalTo(10l));
assertThat(VersionType.FORCE.updateVersion(11, 10), equalTo(10l));
// Old indexing code
// if (index.versionType() == VersionType.INTERNAL) { // internal version type
// updatedVersion = (currentVersion == Versions.NOT_SET || currentVersion == Versions.NOT_FOUND) ? 1 : currentVersion + 1;

View File

@ -241,11 +241,8 @@ public class UpdateTests extends ElasticsearchIntegrationTest {
// external versioning
client().prepareIndex("test", "type", "2").setSource("text", "value").setVersion(10).setVersionType(VersionType.EXTERNAL).get();
assertThrows(client().prepareUpdate("test", "type", "2").setScript("ctx._source.text = 'v2'").setVersion(2).setVersionType(VersionType.EXTERNAL).execute(),
VersionConflictEngineException.class);
ActionRequestValidationException.class);
client().prepareUpdate("test", "type", "2").setScript("ctx._source.text = 'v2'").setVersion(11).setVersionType(VersionType.EXTERNAL).get();
assertThat(client().prepareGet("test", "type", "2").get().getVersion(), equalTo(11l));
// upserts - the combination with versions is a bit weird. Test are here to ensure we do not change our behavior unintentionally
@ -255,9 +252,9 @@ public class UpdateTests extends ElasticsearchIntegrationTest {
assertThat(get.getVersion(), equalTo(1l));
assertThat((String) get.getSource().get("text"), equalTo("v0"));
// With external versions, it means - if object is there with version lower than X, update it or explode. If it is not there, insert with new version.
// With force version
client().prepareUpdate("test", "type", "4").setScript("ctx._source.text = 'v2'").
setVersion(10).setVersionType(VersionType.EXTERNAL).setUpsert("{ \"text\": \"v0\" }").get();
setVersion(10).setVersionType(VersionType.FORCE).setUpsert("{ \"text\": \"v0\" }").get();
get = get("test", "type", "4");
assertThat(get.getVersion(), equalTo(10l));
assertThat((String) get.getSource().get("text"), equalTo("v0"));