diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3b3e7e9d300..4383c3415a8 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -70,7 +70,10 @@ Apache ZooKeeper 3.4.14 Jetty 9.4.14.v20181114 -(No Changes) +New Features +---------------------- + +: SOLR-13320 : add an update param failOnVersionConflicts=false to updates not fail when there is a version conflict (noble) ================== 8.1.0 ================== diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index ed48c758f47..1b7c81b9e12 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -389,6 +389,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd // specified it must exist (versionOnUpdate==1) and it does. } else { + if(cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true) == false) { + return true; + } + throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion); } diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java index 88dc7bb6f3e..02452a1ed9d 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java @@ -41,6 +41,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.index.NoMergePolicyFactory; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.IndexSchema; @@ -1086,7 +1087,30 @@ public class TestInPlaceUpdatesStandalone extends SolrTestCaseJ4 { assertEquals(version1, cmd.prevVersion); } } - + + public void testFailOnVersionConflicts() throws Exception { + + assertU(add(doc("id", "1", "title_s", "first"))); + assertU(commit()); + assertQ(req("q", "title_s:first"), "//*[@numFound='1']"); + assertU(add(doc("id", "1", "title_s", "first1"))); + assertU(commit()); + assertQ(req("q", "title_s:first1"), "//*[@numFound='1']"); + assertFailedU(add(doc("id", "1", "title_s", "first2", "_version_", "-1"))); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("_version_", "-1"); + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "first2"); + SolrInputDocument doc2 = new SolrInputDocument("id", "2", "title_s", "second"); + SolrException ex = expectThrows(SolrException.class, "This should have failed", () -> updateJ(jsonAdd(doc, doc2), params)); + + assertTrue(ex.getMessage().contains("version conflict for")); + params.add(CommonParams.FAIL_ON_VERSION_CONFLICTS, "false"); + updateJ(jsonAdd(doc, doc2), params);//this should not throw any error + assertU(commit()); + assertQ(req("q", "title_s:second"), "//*[@numFound='1']"); + assertQ(req("q", "title_s:first1"), "//*[@numFound='1']");// but the old value exists + assertQ(req("q", "title_s:first2"), "//*[@numFound='0']");// and the new value does not reflect + } /** * Helper method that sets up a req/cmd to run {@link AtomicUpdateDocumentMerger#computeInPlaceUpdatableFields} * on the specified solr input document. diff --git a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc b/solr/solr-ref-guide/src/updating-parts-of-documents.adoc index 8aea928cc60..7e48b369d94 100644 --- a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc +++ b/solr/solr-ref-guide/src/updating-parts-of-documents.adoc @@ -290,6 +290,8 @@ When the client resubmits a changed document to Solr, the `\_version_` can be in * If the content in the `\_version_` field is less than '0' (i.e., '-1'), then the document must *not* exist. In this case, no version matching occurs, but if the document exists, the updates will be rejected. * If the content in the `\_version_` field is equal to '0', then it doesn't matter if the versions match or if the document exists or not. If it exists, it will be overwritten; if it does not exist, it will be added. +When documents are added/updated in batches even a single version conflict may lead to rejecting the entire batch. Use the parameter `failOnVersionConflicts=false` to avoid failure of the entire batch when version constraints fail for one or more documents in a batch. + If the document being updated does not include the `\_version_` field, and atomic updates are not being used, the document will be treated by normal Solr rules, which is usually to discard the previous version. When using Optimistic Concurrency, clients can include an optional `versions=true` request parameter to indicate that the _new_ versions of the documents being added should be included in the response. This allows clients to immediately know what the `\_version_` is of every document added without needing to make a redundant <>. @@ -298,30 +300,35 @@ Following are some examples using `versions=true` in queries: [source,bash] ---- -$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?versions=true' --data-binary ' +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?versions=true&omitHeader=true' --data-binary ' [ { "id" : "aaa" }, { "id" : "bbb" } ]' ---- [source,json] ---- -{"responseHeader":{"status":0,"QTime":6}, - "adds":["aaa",1498562471222312960, - "bbb",1498562471225458688]} +{ + "adds":[ + "aaa",1632740120218042368, + "bbb",1632740120250548224]} ---- In this example, we have added 2 documents "aaa" and "bbb". Because we added `versions=true` to the request, the response shows the document version for each document. [source,bash] ---- -$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?_version_=999999&versions=true' --data-binary ' -[{ "id" : "aaa", - "foo_s" : "update attempt with wrong existing version" }]' +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?_version_=999999&versions=true&omitHeader=true' --data-binary ' + [{ "id" : "aaa", + "foo_s" : "update attempt with wrong existing version" }]' ---- [source,json] ---- -{"responseHeader":{"status":409,"QTime":3}, - "error":{"msg":"version conflict for aaa expected=999999 actual=1498562471222312960", - "code":409}} +{ + "error":{ + "metadata":[ + "error-class","org.apache.solr.common.SolrException", + "root-error-class","org.apache.solr.common.SolrException"], + "msg":"version conflict for aaa expected=999999 actual=1632740120218042368", + "code":409}} ---- @@ -329,44 +336,88 @@ In this example, we've attempted to update document "aaa" but specified the wron [source,bash] ---- -$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?_version_=1498562471222312960&versions=true&commit=true' --data-binary ' +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?_version_=1632740120218042368&versions=true&commit=true&omitHeader=true' --data-binary ' [{ "id" : "aaa", "foo_s" : "update attempt with correct existing version" }]' ---- [source,json] ---- -{"responseHeader":{"status":0,"QTime":5}, - "adds":["aaa",1498562624496861184]} +{ + "adds":[ + "aaa",1632740462042284032]} ---- Now we've sent an update with a value for `\_version_` that matches the value in the index, and it succeeds. Because we included `versions=true` to the update request, the response includes a different value for the `\_version_` field. +[source,bash] +---- +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true&omitHeader=true' --data-binary ' +[{ "id" : "aaa", _version_ : 100, + "foo_s" : "update attempt with wrong existing version embedded in document" }]' +---- +[source,json] +---- +{ + "error":{ + "metadata":[ + "error-class","org.apache.solr.common.SolrException", + "root-error-class","org.apache.solr.common.SolrException"], + "msg":"version conflict for aaa expected=100 actual=1632740462042284032", + "code":409}} +---- + +Now we've sent an update with a value for `\_version_` embedded in the document itself. this request fails because we have specified the wrong version. This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc. [source,bash] ---- -$ curl 'http://localhost:8983/solr/techproducts/query?q=*:*&fl=id,_version_' +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true&omitHeader=true' --data-binary ' +[{ "id" : "aaa", _version_ : 1632740462042284032, + "foo_s" : "update attempt with correct version embedded in document" }]' +---- +[source,json] +---- +{ + "adds":[ + "aaa",1632741942747987968]} +---- + +Now we've sent an update with a value for `\_version_` embedded in the document itself. this request fails because we have specified the wrong version. This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc. + + +[source,bash] +---- +$ curl 'http://localhost:8983/solr/techproducts/query?q=*:*&fl=id,_version_&omitHeader=true' ---- [source,json] ---- { - "responseHeader":{ - "status":0, - "QTime":5, - "params":{ - "fl":"id,_version_", - "q":"*:*"}}, - "response":{"numFound":2,"start":0,"docs":[ - { - "id":"bbb", - "_version_":1498562471225458688}, - { - "id":"aaa", - "_version_":1498562624496861184}] + "response":{"numFound":3,"start":0,"docs":[ + { "_version_":1632740120250548224, + "id":"bbb"}, + { "_version_":1632741942747987968, + "id":"aaa"}] }} + ---- Finally, we can issue a query that requests the `\_version_` field be included in the response, and we can see that for the two documents in our example index. +[source,bash] +---- +$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?versions=true&_version_=-1&failOnVersionConflicts=false&omitHeader=true' --data-binary ' +[ { "id" : "aaa" }, + { "id" : "ccc" } ]' +---- +[source,json] +---- +{ + "adds":[ + "ccc",1632740949182382080]} +---- + +In this example, we have added 2 documents "aaa" and "ccc". As we have specified the parameter `\_version_=-1` , this request should not add the document with the id `aaa` because it already exists. The request succeeds & does not throw any error because the `failOnVersionConflicts=false` parameter is specified. The response shows that only document `ccc` is added and `aaa` is silently ignored. + + For more information, please also see Yonik Seeley's presentation on https://www.youtube.com/watch?v=WYVM6Wz-XTw[NoSQL features in Solr 4] from Apache Lucene EuroCon 2012. == Document Centric Versioning Constraints diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index a13181c53ac..39a02428161 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -289,6 +289,8 @@ public interface CommonParams { String VERSION_FIELD="_version_"; + String FAIL_ON_VERSION_CONFLICTS ="failOnVersionConflicts"; + String ID = "id"; String JSON_MIME = "application/json"; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index 9f5f7f5d1a4..404906408a8 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -305,9 +305,52 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase assertEquals( 1, rsp.getResults().getNumFound() ); } - + @Test + public void testFailOnVersionConflicts() throws Exception { + SolrClient client = getSolrClient(); - /** + // Empty the database... + client.deleteByQuery("*:*");// delete everything! + client.commit(); + + client.request(new UpdateRequest() + .add("id", "id1", "name", "doc1.v1")); + client.commit(); + + QueryResponse rsp = null; + assertResponseValues(client.query(new SolrQuery("id:id1")), "response[0]/name", "doc1.v1"); + + assertResponseValues(client.query(new SolrQuery("*:*").set("sort", "id asc")), + "response[0]/name", "doc1.v1" + ); + + client.request( + new UpdateRequest() + .add("id", "id1", "name", "doc1.v2") + .add("id", "id2", "name", "doc2.v1")); + client.commit(); + assertResponseValues(client.query(new SolrQuery("*:*").set("sort", "id asc")), + "response[0]/name", "doc1.v2", + "response[1]/name", "doc2.v1" + ); + + UpdateRequest add = new UpdateRequest() + .add("id", "id1", "name", "doc1.v3") + .add("id", "id3", "name", "doc3.v1"); + add.setParam(CommonParams.FAIL_ON_VERSION_CONFLICTS, "false"); + add.setParam(CommonParams.VERSION_FIELD, "-1"); + client.request(add); + client.commit(); + + assertResponseValues(client.query(new SolrQuery("*:*").set("sort", "id asc")), + "response[0]/name", "doc1.v2" , + "response[1]/name", "doc2.v1" , + "response[2]/name", "doc3.v1"); + + } + + + /** * Get empty results */ @Test diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index c286e42d2d9..b6541abdf35 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -100,6 +100,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.LBHttpSolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.IpTables; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -119,6 +120,7 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.XML; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoresLocator; @@ -1758,7 +1760,16 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase { } - + public static void assertResponseValues(SolrResponseBase rsp, Object... assertions) { + Map values = Utils.makeMap(assertions); + values.forEach((s, o) -> { + if (o instanceof String) { + assertEquals(o, rsp.getResponse()._getStr(s, null)); + } else { + assertEquals(o, rsp.getResponse()._get(s, null)); + } + }); + } public Map indexDocs(List descriptor, Map model, int nDocs) throws Exception { if (model == null) { model = new LinkedHashMap<>();