mirror of https://github.com/apache/lucene.git
SOLR-9034: fix atomic updates for copyField w/ docValues
This commit is contained in:
parent
2c66d4b046
commit
c897917c71
|
@ -159,6 +159,9 @@ Bug Fixes
|
||||||
* SOLR-9046: Fix solr.cmd that wrongly assumes Jetty will always listen on 0.0.0.0.
|
* SOLR-9046: Fix solr.cmd that wrongly assumes Jetty will always listen on 0.0.0.0.
|
||||||
(Bram Van Dam, Uwe Schindler)
|
(Bram Van Dam, Uwe Schindler)
|
||||||
|
|
||||||
|
* SOLR-9034: Atomic updates failed to work when there were copyField targets that had docValues
|
||||||
|
enabled. (Karthik Ramachandran, Ishan Chattopadhyaya, yonik)
|
||||||
|
|
||||||
Optimizations
|
Optimizations
|
||||||
----------------------
|
----------------------
|
||||||
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
|
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
|
||||||
|
|
|
@ -342,7 +342,7 @@ public class RealTimeGetComponent extends SearchComponent
|
||||||
if (docid < 0) return null;
|
if (docid < 0) return null;
|
||||||
Document luceneDocument = searcher.doc(docid);
|
Document luceneDocument = searcher.doc(docid);
|
||||||
sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
|
sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
|
||||||
searcher.decorateDocValueFields(sid, docid, searcher.getNonStoredDVs(false));
|
searcher.decorateDocValueFields(sid, docid, searcher.getNonStoredDVsWithoutCopyTargets());
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (searcherHolder != null) {
|
if (searcherHolder != null) {
|
||||||
|
|
|
@ -183,6 +183,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
||||||
/** Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema. */
|
/** Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema. */
|
||||||
private final Set<String> nonStoredDVsUsedAsStored;
|
private final Set<String> nonStoredDVsUsedAsStored;
|
||||||
|
|
||||||
|
/** Contains the names/patterns of all docValues=true,stored=false fields, excluding those that are copyField targets in the schema. */
|
||||||
|
private final Set<String> nonStoredDVsWithoutCopyTargets;
|
||||||
|
|
||||||
private Collection<String> storedHighlightFieldNames;
|
private Collection<String> storedHighlightFieldNames;
|
||||||
private DirectoryFactory directoryFactory;
|
private DirectoryFactory directoryFactory;
|
||||||
|
|
||||||
|
@ -359,6 +362,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
||||||
|
|
||||||
final Set<String> nonStoredDVsUsedAsStored = new HashSet<>();
|
final Set<String> nonStoredDVsUsedAsStored = new HashSet<>();
|
||||||
final Set<String> allNonStoredDVs = new HashSet<>();
|
final Set<String> allNonStoredDVs = new HashSet<>();
|
||||||
|
final Set<String> nonStoredDVsWithoutCopyTargets = new HashSet<>();
|
||||||
|
|
||||||
this.fieldInfos = leafReader.getFieldInfos();
|
this.fieldInfos = leafReader.getFieldInfos();
|
||||||
for (FieldInfo fieldInfo : fieldInfos) {
|
for (FieldInfo fieldInfo : fieldInfos) {
|
||||||
final SchemaField schemaField = schema.getFieldOrNull(fieldInfo.name);
|
final SchemaField schemaField = schema.getFieldOrNull(fieldInfo.name);
|
||||||
|
@ -367,11 +372,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
||||||
nonStoredDVsUsedAsStored.add(fieldInfo.name);
|
nonStoredDVsUsedAsStored.add(fieldInfo.name);
|
||||||
}
|
}
|
||||||
allNonStoredDVs.add(fieldInfo.name);
|
allNonStoredDVs.add(fieldInfo.name);
|
||||||
|
if (!schema.isCopyFieldTarget(schemaField)) {
|
||||||
|
nonStoredDVsWithoutCopyTargets.add(fieldInfo.name);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored);
|
this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored);
|
||||||
this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
|
this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
|
||||||
|
this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets);
|
||||||
|
|
||||||
// We already have our own filter cache
|
// We already have our own filter cache
|
||||||
setQueryCache(null);
|
setQueryCache(null);
|
||||||
|
@ -876,6 +885,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
||||||
return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs;
|
return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns an unmodifiable set of names of non-stored docValues fields, except those that are targets of a copy field.
|
||||||
|
*/
|
||||||
|
public Set<String> getNonStoredDVsWithoutCopyTargets() {
|
||||||
|
return nonStoredDVsWithoutCopyTargets;
|
||||||
|
}
|
||||||
|
|
||||||
/* ********************** end document retrieval *************************/
|
/* ********************** end document retrieval *************************/
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
|
@ -648,20 +648,30 @@
|
||||||
<dynamicField name="*_sev_enum" type="severityType" indexed="true" stored="true" docValues="true" multiValued="true" />
|
<dynamicField name="*_sev_enum" type="severityType" indexed="true" stored="true" docValues="true" multiValued="true" />
|
||||||
|
|
||||||
<!-- With DocValues=true -->
|
<!-- With DocValues=true -->
|
||||||
<dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true"/>
|
<dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true" />
|
||||||
<dynamicField name="*_l_dv" type="long" indexed="true" stored="true" docValues="true"/>
|
<dynamicField name="*_l_dv" type="long" indexed="true" stored="true" docValues="true" />
|
||||||
<dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true"/>
|
<dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true" />
|
||||||
<dynamicField name="*_d_dv" type="double" indexed="true" stored="true" docValues="true"/>
|
<dynamicField name="*_d_dv" type="double" indexed="true" stored="true" docValues="true" />
|
||||||
<dynamicField name="*_dt_dv" type="date" indexed="true" stored="true" docValues="true"/>
|
<dynamicField name="*_dt_dv" type="date" indexed="true" stored="true" docValues="true" />
|
||||||
<dynamicField name="*_f1_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
<dynamicField name="*_f1_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="false" />
|
||||||
|
|
||||||
<!-- Non-stored, DocValues=true -->
|
<!-- Non-stored, DocValues=true -->
|
||||||
<dynamicField name="*_i_dvo" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
|
<dynamicField name="*_i_dvo" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
|
||||||
<dynamicField name="*_d_dvo" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
|
<dynamicField name="*_d_dvo" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
|
||||||
<dynamicField name="*_s_dvo" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
|
<dynamicField name="*_s_dvo" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
|
||||||
<dynamicField name="*_ii_dvo" multiValued="true" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
|
<dynamicField name="*_ii_dvo" multiValued="true" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
|
||||||
<dynamicField name="*_dd_dvo" multiValued="true" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
|
<dynamicField name="*_dd_dvo" multiValued="true" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
|
||||||
|
|
||||||
|
<!-- Non-stored, DocValues=true, useDocValuesAsStored=false -->
|
||||||
|
<field name="single_i_dvn" multiValued="false" type="int" indexed="true" stored="true" />
|
||||||
|
<field name="single_d_dvn" multiValued="false" type="double" indexed="true" stored="true" />
|
||||||
|
<field name="single_s_dvn" multiValued="false" type="string" indexed="true" stored="true" />
|
||||||
|
<field name="copy_single_i_dvn" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
|
||||||
|
<field name="copy_single_d_dvn" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
|
||||||
|
<field name="copy_single_s_dvn" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
|
||||||
|
<copyField source="single_i_dvn" dest="copy_single_i_dvn" />
|
||||||
|
<copyField source="single_d_dvn" dest="copy_single_d_dvn" />
|
||||||
|
<copyField source="single_s_dvn" dest="copy_single_s_dvn" />
|
||||||
</fields>
|
</fields>
|
||||||
|
|
||||||
<defaultSearchField>text</defaultSearchField>
|
<defaultSearchField>text</defaultSearchField>
|
||||||
|
|
|
@ -43,6 +43,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
|
||||||
h.update("<delete><query>*:*</query></delete>");
|
h.update("<delete><query>*:*</query></delete>");
|
||||||
assertU(commit());
|
assertU(commit());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRemove() throws Exception {
|
public void testRemove() throws Exception {
|
||||||
SolrInputDocument doc;
|
SolrInputDocument doc;
|
||||||
|
@ -1065,6 +1066,49 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
|
||||||
"/response/docs/[0]/multi_ii_dvo/[0]==100",
|
"/response/docs/[0]/multi_ii_dvo/[0]==100",
|
||||||
"/response/docs/[0]/multi_ii_dvo/[1]==" + Integer.MAX_VALUE);
|
"/response/docs/[0]/multi_ii_dvo/[1]==" + Integer.MAX_VALUE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAtomicUpdatesOnNonStoredDocValuesCopyField() throws Exception {
|
||||||
|
assertU(adoc(sdoc("id", 101, "title", "title2", "single_i_dvn", 100)));
|
||||||
|
assertU(adoc(sdoc("id", 102, "title", "title3", "single_d_dvn", 3.14)));
|
||||||
|
assertU(adoc(sdoc("id", 103, "single_s_dvn", "abc", "single_i_dvn", 1)));
|
||||||
|
assertU(commit());
|
||||||
|
|
||||||
|
// Do each one twice... the first time it will be retrieved from the index, and the second time from the transaction log.
|
||||||
|
for (int i=0; i<2; i++) {
|
||||||
|
assertU(adoc(sdoc("id", 101, "title", ImmutableMap.of("set", "newtitle2"),
|
||||||
|
"single_i_dvn", ImmutableMap.of("inc", 1))));
|
||||||
|
assertU(adoc(sdoc("id", 102, "title", ImmutableMap.of("set", "newtitle3"),
|
||||||
|
"single_d_dvn", ImmutableMap.of("inc", 1))));
|
||||||
|
assertU(adoc(sdoc("id", 103, "single_i_dvn", ImmutableMap.of("inc", 1))));
|
||||||
|
}
|
||||||
|
assertU(commit());
|
||||||
|
|
||||||
|
assertJQ(req("q", "id:101"),
|
||||||
|
"/response/docs/[0]/id==101",
|
||||||
|
"/response/docs/[0]/title/[0]=='newtitle2'",
|
||||||
|
"/response/docs/[0]/single_i_dvn==102");
|
||||||
|
|
||||||
|
assertJQ(req("q", "id:102"),
|
||||||
|
1e-4,
|
||||||
|
"/response/docs/[0]/id==102",
|
||||||
|
"/response/docs/[0]/title/[0]=='newtitle3'",
|
||||||
|
"/response/docs/[0]/single_d_dvn==5.14");
|
||||||
|
|
||||||
|
assertJQ(req("q", "id:103"),
|
||||||
|
"/response/docs/[0]/id==103",
|
||||||
|
"/response/docs/[0]/single_s_dvn=='abc'",
|
||||||
|
"/response/docs/[0]/single_i_dvn==3");
|
||||||
|
|
||||||
|
// test that non stored docvalues was carried forward for a non-docvalue update
|
||||||
|
assertU(adoc(sdoc("id", 103, "single_s_dvn", ImmutableMap.of("set", "abcupdate"),
|
||||||
|
"single_i_dvn", ImmutableMap.of("set", 5))));
|
||||||
|
assertU(commit());
|
||||||
|
assertJQ(req("q", "id:103"),
|
||||||
|
"/response/docs/[0]/id==103",
|
||||||
|
"/response/docs/[0]/single_s_dvn=='abcupdate'",
|
||||||
|
"/response/docs/[0]/single_i_dvn==5");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testInvalidOperation() {
|
public void testInvalidOperation() {
|
||||||
|
|
Loading…
Reference in New Issue