Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues

This commit is contained in:
yonik 2016-03-25 10:18:31 -04:00
parent c498fcdc43
commit 6f60ac21f3
5 changed files with 93 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -361,6 +361,9 @@ Bug Fixes
* SOLR-8891: Fix StrField.toObject and toExternal to work with docValue IndexableField
instances. (yonik)
* SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues.
(Ishan Chattopadhyaya, yonik)
Optimizations
----------------------
* SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been

13
solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
@ -258,6 +259,7 @@ public class RealTimeGetComponent extends SearchComponent
if (docid < 0) continue;
Document luceneDocument = searcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames());
SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
searcher.decorateDocValueFields(doc, docid, searcher.getNonStoredDVs(true));
if( transformer != null ) {
transformer.transform(doc, docid, 0);
}
@ -408,8 +410,15 @@ public class RealTimeGetComponent extends SearchComponent
// copy the stored fields only
Document out = new Document();
for (IndexableField f : doc.getFields()) {
if (f.fieldType().stored() ) {
out.add((IndexableField) f);
if (f.fieldType().stored()) {
out.add(f);
} else if (f.fieldType().docValuesType() != DocValuesType.NONE) {
SchemaField schemaField = schema.getFieldOrNull(f.name());
if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored()) {
out.add(f);
}
} else {
log.debug("Don't know how to handle field " + f);
}
}

1
solr/core/src/test-files/solr/collection1/conf/schema15.xml
View File

@ -530,6 +530,7 @@
<dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_ll" type="long" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_tt" type="text" indexed="true" stored="true"/>
<dynamicField name="*_ws" type="nametext" indexed="true" stored="true"/>

51
solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
View File

@ -110,7 +110,7 @@
<!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
or Solr won't start. _version_ and update log are required for SolrCloud
-->
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="_version_" type="long" indexed="false" stored="false" docValues="true" />
<!-- points to the root document of a block of nested documents. Required for nested
document support, may be removed otherwise
@ -199,27 +199,48 @@
RESTRICTION: the glob-like pattern in the name attribute must have
a "*" only at the start or the end. -->
<!-- docvalues and stored are exclusive -->
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_is" type="int" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_id" type="int" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_ids" type="int" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_id" type="int" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_ids" type="int" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_s" type="string" indexed="true" stored="true" />
<dynamicField name="*_s1" type="string" indexed="true" stored="true" />
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_sd" type="string" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_sds" type="string" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_sd" type="string" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_sds" type="string" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_ls" type="long" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_ld" type="long" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_lds" type="long" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_ld" type="long" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_lds" type="long" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_fs" type="float" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_fd" type="float" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_fds" type="float" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_fd" type="float" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_fds" type="float" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_dd" type="double" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_dds" type="double" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_dd" type="double" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_dds" type="double" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_dtd" type="date" indexed="true" stored="false" docValues="true" />
<dynamicField name="*_dtds" type="date" indexed="true" stored="false" multiValued="true" docValues="true" />
<!-- docvalues and stored (S suffix) -->
<dynamicField name="*_idS" type="int" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_idsS" type="int" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_sdS" type="string" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_sdsS" type="string" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_ldS" type="long" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_ldsS" type="long" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_fdS" type="float" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_fdsS" type="float" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_ddS" type="double" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_ddsS" type="double" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_dtdS" type="date" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_dtdsS" type="date" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_bs" type="boolean" indexed="true" stored="true" multiValued="true"/>
@ -230,10 +251,6 @@
<!-- Type used to index the lat and lon components for the "location" FieldType -->
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_dtd" type="date" indexed="true" stored="true" docValues="true" />
<dynamicField name="*_dtds" type="date" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_p" type="location" indexed="true" stored="true"/>
<!-- some trie-coded dynamic fields for faster range queries -->
@ -258,8 +275,8 @@
<!-- needed by dedup config -->
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
</fields>
@ -695,7 +712,7 @@
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
<fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
<fieldtype name="ignored" stored="false" indexed="false" docValues="false" multiValued="true" class="solr.StrField" />
<!-- This point type indexes the coordinates as separate fields (subFields)
If subFieldType is defined, it references a type, and a dynamic field

49
solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
View File

@ -43,7 +43,7 @@ public class TestRealTimeGet extends TestRTGBase {
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig-tlog.xml","schema15.xml");
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@ -52,12 +52,22 @@ public class TestRealTimeGet extends TestRTGBase {
clearIndex();
assertU(commit());
assertU(adoc("id","1"));
assertU(adoc("id","1",
"a_f","-1.5", "a_fd","-1.5", "a_fdS","-1.5", "a_fs","1.0","a_fs","2.5", "a_fds","1.0","a_fds","2.5", "a_fdsS","1.0","a_fdsS","2.5",
"a_d","-1.2E99", "a_dd","-1.2E99", "a_ddS","-1.2E99", "a_ds","1.0","a_ds","2.5", "a_dds","1.0","a_dds","2.5", "a_ddsS","1.0","a_ddsS","2.5",
"a_i","-1", "a_id","-1", "a_idS","-1", "a_is","1","a_is","2", "a_ids","1","a_ids","2", "a_idsS","1","a_idsS","2",
"a_l","-9999999999", "a_ld","-9999999999", "a_ldS","-9999999999", "a_ls","1","a_ls","9999999999", "a_lds","1","a_lds","9999999999", "a_ldsS","1","a_ldsS","9999999999"
));
assertJQ(req("q","id:1")
,"/response/numFound==0"
);
assertJQ(req("qt","/get", "id","1", "fl","id")
,"=={'doc':{'id':'1'}}"
assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS a_fs,a_fds,a_fdsS, a_d,a_dd,a_ddS, a_ds,a_dds,a_ddsS, a_i,a_id,a_idS a_is,a_ids,a_idsS, a_l,a_ld,a_ldS a_ls,a_lds,a_ldsS")
,"=={'doc':{'id':'1'" +
", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5, a_fs:[1.0,2.5], a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]" +
", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99, a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]" +
", a_i:-1, a_id:-1, a_idS:-1, a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]" +
", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999, a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]" +
" }}"
);
assertJQ(req("qt","/get","ids","1", "fl","id")
,"=={" +
@ -72,6 +82,17 @@ public class TestRealTimeGet extends TestRTGBase {
assertJQ(req("q","id:1")
,"/response/numFound==1"
);
// a cut-n-paste of the first big query, but this time it will be retrieved from the index rather than the transaction log
assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS a_fs,a_fds,a_fdsS, a_d,a_dd,a_ddS, a_ds,a_dds,a_ddsS, a_i,a_id,a_idS a_is,a_ids,a_idsS, a_l,a_ld,a_ldS a_ls,a_lds,a_ldsS")
,"=={'doc':{'id':'1'" +
", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5, a_fs:[1.0,2.5], a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]" +
", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99, a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]" +
", a_i:-1, a_id:-1, a_idS:-1, a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]" +
", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999, a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]" +
" }}"
);
assertJQ(req("qt","/get","id","1", "fl","id")
,"=={'doc':{'id':'1'}}"
);
@ -101,7 +122,7 @@ public class TestRealTimeGet extends TestRTGBase {
assertJQ(req("qt","/get","id","10", "fl","id")
,"=={'doc':{'id':'10'}}"
);
assertU(delQ("id:10 abcdef"));
assertU(delQ("id:10 foo_s:abcdef"));
assertJQ(req("qt","/get","id","10")
,"=={'doc':null}"
);
@ -109,6 +130,24 @@ public class TestRealTimeGet extends TestRTGBase {
,"=={'doc':{'id':'11'}}"
);
// multivalued field
assertU(adoc("id","12", "val_ls","1", "val_ls","2"));
assertJQ(req("q","id:12")
,"/response/numFound==0"
);
assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
);
assertU(commit());
assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
);
assertJQ(req("q","id:12")
,"/response/numFound==1"
);
SolrQueryRequest req = req();
RefCounted<SolrIndexSearcher> realtimeHolder = req.getCore().getRealtimeSearcher();