Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12378: Support missing versionField on indexed docs in DocBasedVersionConstraintsURP.

This commit is contained in:
markrmiller 2018-05-23 09:53:45 -05:00
parent 53a3de3b98
commit 48bd259516
6 changed files with 103 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -119,6 +119,9 @@ New Features
* SOLR-9480: A new 'relatedness()' aggregate function for JSON Faceting to enable building Semantic
Knowledge Graphs. (Trey Grainger, hossman)
* SOLR-12378: Support missing versionField on indexed docs in DocBasedVersionConstraintsURP.
(Oliver Bates, Michael Braun via Mark Miller)
Bug Fixes
----------------------

19
solr/core/src/java/org/apache/solr/update/processor/DocBasedVersionConstraintsProcessor.java
View File

@ -35,7 +35,6 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.RealTimeGetComponent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
@ -60,6 +59,7 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
private final SchemaField[] userVersionFields;
private final SchemaField solrVersionField;
private final boolean ignoreOldUpdates;
private final boolean supportMissingVersionOnOldDocs;
private final String[] deleteVersionParamNames;
private final SolrCore core;
@ -72,13 +72,14 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
public DocBasedVersionConstraintsProcessor(List<String> versionFields,
boolean ignoreOldUpdates,
List<String> deleteVersionParamNames,
boolean supportMissingVersionOnOldDocs,
boolean useFieldCache,
SolrQueryRequest req,
SolrQueryResponse rsp,
UpdateRequestProcessor next ) {
super(next);
this.ignoreOldUpdates = ignoreOldUpdates;
this.deleteVersionParamNames = deleteVersionParamNames.toArray(EMPTY_STR_ARR);
this.supportMissingVersionOnOldDocs = supportMissingVersionOnOldDocs;
this.core = req.getCore();
this.versionFieldNames = versionFields.toArray(EMPTY_STR_ARR);
IndexSchema schema = core.getLatestSchema();
@ -123,10 +124,10 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
return rawValue;
}
private static Object[] convertFieldValuesUsingType(Object[] rawValues, SchemaField[] fields) {
private Object[] convertFieldValuesUsingType(Object[] rawValues) {
Object[] returnArr = new Object[rawValues.length];
for (int i = 0; i < returnArr.length; i++) {
returnArr[i] = convertFieldValueUsingType(rawValues[i], fields[i]);
returnArr[i] = convertFieldValueUsingType(rawValues[i], userVersionFields[i]);
}
return returnArr;
}
@ -145,7 +146,7 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
assert null != indexedDocId;
assert null != newUserVersions;
newUserVersions = convertFieldValuesUsingType(newUserVersions, userVersionFields);
newUserVersions = convertFieldValuesUsingType(newUserVersions);
final DocFoundAndOldUserAndSolrVersions docFoundAndOldUserVersions;
if (useFieldCache) {
@ -165,11 +166,13 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
return versionInUpdateIsAcceptable(newUserVersions, oldUserVersions);
}
private static void validateUserVersions(Object[] userVersions, String[] fieldNames, String errorMessage) {
private void validateUserVersions(Object[] userVersions, String[] fieldNames, String errorMessage) {
assert userVersions.length == fieldNames.length;
for (int i = 0; i < userVersions.length; i++) {
Object userVersion = userVersions[i];
if ( null == userVersion) {
if (supportMissingVersionOnOldDocs && null == userVersion) {
userVersions[i] = (Comparable<Object>) o -> -1;
} else if (null == userVersion) {
// could happen if they turn this feature on after building an index
// w/o the versionField, or if validating a new doc, not present.
throw new SolrException(SERVER_ERROR, errorMessage + fieldNames[i]);
@ -320,7 +323,7 @@ public class DocBasedVersionConstraintsProcessor extends UpdateRequestProcessor
* @return True if acceptable, false if not.
*/
protected boolean newUpdateComparePasses(Comparable newUserVersion, Comparable oldUserVersion, String userVersionFieldName) {
return newUserVersion.compareTo(oldUserVersion) > 0;
return oldUserVersion.compareTo(newUserVersion) < 0;
}
private static Object[] getObjectValues(LeafReaderContext segmentContext,

20
solr/core/src/java/org/apache/solr/update/processor/DocBasedVersionConstraintsProcessorFactory.java
View File

@ -80,6 +80,11 @@ import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
* document version that is not great enough to be silently ignored (and return
* a status 200 to the client) instead of generating a 409 Version Conflict error.
* </li>
*
* <li><code>supportMissingVersionOnOldDocs</code> - This boolean parameter defaults to
* <code>false</code>, but if set to <code>true</code> allows any documents written *before*
* this feature is enabled and which are missing the versionField to be overwritten.
* </li>
* </ul>
* @since 4.6.0
*/
@ -90,6 +95,7 @@ public class DocBasedVersionConstraintsProcessorFactory extends UpdateRequestPro
private List<String> versionFields = null;
private List<String> deleteVersionParamNames = Collections.emptyList();
private boolean useFieldCache;
private boolean supportMissingVersionOnOldDocs = false;
@Override
public void init( NamedList args ) {
@ -129,6 +135,17 @@ public class DocBasedVersionConstraintsProcessorFactory extends UpdateRequestPro
}
ignoreOldUpdates = (Boolean) tmp;
}
// optional - defaults to false
tmp = args.remove("supportMissingVersionOnOldDocs");
if (null != tmp) {
if (! (tmp instanceof Boolean) ) {
throw new SolrException(SERVER_ERROR,
"'supportMissingVersionOnOldDocs' must be configured as a <bool>");
}
supportMissingVersionOnOldDocs = ((Boolean)tmp).booleanValue();
}
super.init(args);
}
@ -139,8 +156,9 @@ public class DocBasedVersionConstraintsProcessorFactory extends UpdateRequestPro
return new DocBasedVersionConstraintsProcessor(versionFields,
ignoreOldUpdates,
deleteVersionParamNames,
supportMissingVersionOnOldDocs,
useFieldCache,
req, rsp, next);
req, next);
}
@Override

25
solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml
View File

@ -126,5 +126,30 @@
<requestHandler name="/select" class="solr.SearchHandler">
</requestHandler>
<updateRequestProcessorChain name="no-external-version">
<!-- this chain lets us index docs without a version field. It's to let us test the optional
'supportMissingVersionOnOldDocs' param in the 'external-version-support-missing' chain
below.
-->
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<updateRequestProcessorChain name="external-version-support-missing">
<!-- this chain sets the supportMissingVersionOnOldDocs param to true so that we can update
docs that were originally indexed without an external version field, e.g. by using the
'no-external-version' chain.
-->
<!-- process the external version constraint, ignoring any updates that
don't satisfy the constraint -->
<processor class="solr.DocBasedVersionConstraintsProcessorFactory">
<bool name="ignoreOldUpdates">true</bool>
<str name="versionField">my_version_l</str>
<str name="deleteVersionParam">del_version</str>
<bool name="supportMissingVersionOnOldDocs">true</bool>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
</config>

44
solr/core/src/test/org/apache/solr/update/TestDocBasedVersionConstraints.java
View File

@ -479,6 +479,50 @@ public class TestDocBasedVersionConstraints extends SolrTestCaseJ4 {
}
}
public void testMissingVersionOnOldDocs() throws Exception {
String version = "2";
// Write one doc with version, one doc without version using the "no version" chain
updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + version + "}]"),
params("update.chain", "no-external-version"));
updateJ(json("[{\"id\": \"b\", \"name\": \"b1\"}]"), params("update.chain", "no-external-version"));
assertU(commit());
assertJQ(req("q","*:*"), "/response/numFound==2");
assertJQ(req("q","id:a"), "/response/numFound==1");
assertJQ(req("q","id:b"), "/response/numFound==1");
// Try updating both with a new version and using the enforced version chain, expect id=b to fail bc old
// doc is missing the version field
version = "3";
updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + version + "}]"),
params("update.chain", "external-version-constraint"));
try {
updateJ(json("[{\"id\": \"b\", \"name\": \"b1\", \"my_version_l\": " + version + "}]"),
params("update.chain", "external-version-constraint"));
fail("Update to id=b should have failed because existing doc is missing version field");
} catch (final SolrException ex) {
// expected
assertEquals("Doc exists in index, but has null versionField: my_version_l", ex.getMessage());
}
assertU(commit());
assertJQ(req("q","*:*"), "/response/numFound==2");
assertJQ(req("qt","/get", "id", "a", "fl", "id,my_version_l"), "=={'doc':{'id':'a', 'my_version_l':3}}"); // version changed to 3
assertJQ(req("qt","/get", "id", "b", "fl", "id,my_version_l"), "=={'doc':{'id':'b'}}"); // no version, because update failed
// Try to update again using the external version enforcement, but allowing old docs to not have the version
// field. Expect id=a to fail because version is lower, expect id=b to succeed.
version = "1";
updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + version + "}]"),
params("update.chain", "external-version-support-missing"));
System.out.println("send b");
updateJ(json("[{\"id\": \"b\", \"name\": \"b1\", \"my_version_l\": " + version + "}]"),
params("update.chain", "external-version-support-missing"));
assertU(commit());
assertJQ(req("q","*:*"), "/response/numFound==2");
assertJQ(req("qt","/get", "id", "a", "fl", "id,my_version_l"), "=={'doc':{'id':'a', 'my_version_l':3}}");
assertJQ(req("qt","/get", "id", "b", "fl", "id,my_version_l"), "=={'doc':{'id':'b', 'my_version_l':1}}");
}
private Callable<Object> delayedAdd(final String... fields) {
return Executors.callable(() -> {
// log.info("ADDING DOC: " + adoc(fields));

1
solr/solr-ref-guide/src/updating-parts-of-documents.adoc
View File

@ -295,5 +295,6 @@ The `\_version_` field used by Solr for its normal optimistic concurrency also h
The value of this configuration option should be the name of a request parameter that the processor will now consider mandatory for all attempts to Delete By Id, and must be be used by clients to specify a value for the `versionField` which is greater then the existing value of the document to be deleted.
When using this request param, any Delete By Id command with a high enough document version number to succeed will be internally converted into an Add Document command that replaces the existing document with a new one which is empty except for the Unique Key and `versionField` to keeping a record of the deleted version so future Add Document commands will fail if their "new" version is not high enough.
If `versionField` is specified as a list, then this parameter too must be specified as a comma delimited list of the same size so that the parameters correspond with the fields.
* `supportMissingVersionOnOldDocs` - This boolean parameter defaults to `false`, but if set to `true` allows any documents written *before* this feature is enabled and which are missing the versionField to be overwritten.
Please consult the {solr-javadocs}/solr-core/org/apache/solr/update/processor/DocBasedVersionConstraintsProcessorFactory.html[DocBasedVersionConstraintsProcessorFactory javadocs] and https://git1-us-west.apache.org/repos/asf?p=lucene-solr.git;a=blob;f=solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml;hb=HEAD[test solrconfig.xml file] for additional information and example usages.