Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10072: The test TestSelectiveWeightCreation appears to be unreliable.

This commit is contained in:
markrmiller 2017-02-12 17:04:23 -05:00
parent b5c7bd1d10
commit 387c8e9c2d
7 changed files with 76 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -195,6 +195,8 @@ Other Changes
* SOLR-9914: SimpleFacets: refactor "contains" check into "SubstringBytesRefFilter" class.
(Jonny Marks, David Smiley, Christine Poerschke)
* SOLR-10072: The test TestSelectiveWeightCreation appears to be unreliable. (Michael Nilsson via Mark Miller)
================== 6.4.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

6
solr/contrib/ltr/src/test-files/featureExamples/external_features.json
View File

@ -5,11 +5,11 @@
"q" : "{!terms f=title}${user_query}"
}
}, {
"name" : "confidence",
"class" : "org.apache.solr.ltr.feature.ValueFeature",
"name" : "docPopularity",
"class" : "org.apache.solr.ltr.feature.FieldValueFeature",
"store": "fstore2",
"params" : {
"value" : "${myconf}"
"field" : "popularity"
}
}, {
"name":"originalScore",

4
solr/contrib/ltr/src/test-files/modelExamples/external_model.json
View File

@ -2,11 +2,11 @@
"class":"org.apache.solr.ltr.model.LinearModel",
"name":"externalmodel",
"features":[
{ "name": "matchedTitle"}
{ "name": "titlePhraseMatch"}
],
"params":{
"weights": {
"matchedTitle": 0.999
"titlePhraseMatch": 0.555
}
}
}

6
solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json → solr/contrib/ltr/src/test-files/modelExamples/external_model2.json
View File

@ -1,13 +1,13 @@
{
"class":"org.apache.solr.ltr.model.LinearModel",
"name":"externalmodelstore",
"name":"externalmodel2",
"store": "fstore2",
"features":[
{ "name": "confidence"}
{ "name": "docPopularity"}
],
"params":{
"weights": {
"confidence": 0.999
"docPopularity": 0.5
}
}
}

24
solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
View File

@ -25,21 +25,15 @@ public class TestParallelWeightCreation extends TestRerankBase{
public void testLTRScoringQueryParallelWeightCreationResultOrder() throws Exception {
setuptest("solrconfig-ltr_Th10_10.xml", "schema.xml");
assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity",
"1"));
assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity",
"2"));
assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity",
"3"));
assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity",
"4"));
assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity",
"5"));
assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", "1"));
assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2"));
assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3"));
assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4"));
assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5"));
assertU(commit());
loadFeatures("external_features.json");
loadModels("external_model.json");
loadModels("external_model_store.json");
// check to make sure that the order of results will be the same when using parallel weight creation
final SolrQuery query = new SolrQuery();
@ -47,10 +41,10 @@ public class TestParallelWeightCreation extends TestRerankBase{
query.add("fl", "*,score");
query.add("rows", "4");
query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
aftertest();
}

79
solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
View File

@ -106,21 +106,16 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
public static void before() throws Exception {
setuptest(false);
assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity",
"1"));
assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity",
"2"));
assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity",
"3"));
assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity",
"4"));
assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity",
"5"));
assertU(adoc("id", "1", "title", "w3 w1", "description", "w1", "popularity", "1"));
assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2"));
assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3"));
assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4"));
assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5"));
assertU(commit());
loadFeatures("external_features.json");
loadModels("external_model.json");
loadModels("external_model_store.json");
loadModels("external_model2.json");
}
@AfterClass
@ -134,14 +129,14 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
doc.add(newStringField("id", "10", Field.Store.YES));
doc.add(newTextField("field", "wizard the the the the the oz",
Field.Store.NO));
doc.add(new FloatDocValuesField("final-score", 1.0f));
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
doc.add(newStringField("id", "11", Field.Store.YES));
// 1 extra token, but wizard and oz are close;
doc.add(newTextField("field", "wizard oz the the the the the the",
Field.Store.NO));
@ -159,8 +154,8 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
// first run the standard query
final TopDocs hits = searcher.search(bqBuilder.build(), 10);
assertEquals(2, hits.totalHits);
assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id"));
assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id"));
assertEquals("10", searcher.doc(hits.scoreDocs[0].doc).get("id"));
assertEquals("11", searcher.doc(hits.scoreDocs[1].doc).get("id"));
List<Feature> features = makeFeatures(new int[] {0, 1, 2});
final List<Feature> allFeatures = makeFeatures(new int[] {0, 1, 2, 3, 4, 5,
@ -206,7 +201,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
}
assertEquals(validFeatures, allFeatures.size());
assertU(delI("0"));assertU(delI("1"));
assertU(delI("10"));assertU(delI("11"));
r.close();
dir.close();
}
@ -216,41 +211,49 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception {
final String docs0fv = FeatureLoggerTestUtils.toFeatureVector(
"matchedTitle","1.0", "titlePhraseMatch","0.40254828");
"matchedTitle","1.0", "titlePhraseMatch","0.6103343");
final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector(
"popularity","3.0", "originalScore","1.0");
// extract all features in externalmodel's store (default store)
// rerank using externalmodel (default store)
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.add("fl", "*,score");
query.add("rows", "4");
query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
query.add("fl", "fv:[fv]");
query.add("fl", "*,score,fv:[fv]");
query.add("rows", "5");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); // extract all features in default store
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
// extract all features from fstore4
// rerank using externalmodel (default store)
query.remove("fl");
query.remove("rq");
query.add("fl", "*,score");
query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
// extract all features from fstore4
// rerank using externalmodel2 (fstore2)
query.remove("fl");
query.remove("rq");
query.add("fl", "*,score");
query.add("rq", "{!ltr reRankDocs=4 model=externalmodelstore efi.user_query=w3 efi.myconf=0.8}");
query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); // score using fstore2 used by externalmodelstore
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7992");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel2 efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='5'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==2.5");
}
}

32
solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
View File

@ -65,24 +65,24 @@ public class TestExternalFeatures extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==1.0");
query.add("fl", "[fv]");
query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
query.remove("fl");
query.add("fl", "*,score,[fv]");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0");
// Adding an efi in the transformer should not affect the rq ranking with a
// different value for efi of the same parameter
query.remove("fl");
query.add("fl", "id,[fv efi.user_query=w2]");
query.add("fl", "*,score,[fv efi.user_query=w2]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0");
}
@Test
@ -92,7 +92,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("fl", "*,score,fv:[fv]");
query.add("rows", "1");
// Stopword only query passed in
query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query='a'}");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a'}");
final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(
"matchedTitle","0.0",
@ -112,7 +112,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("rows", "1");
final String docs0fv_csv = FeatureLoggerTestUtils.toFeatureVector(
"confidence","2.3", "originalScore","1.0");
"occurrences","2.3", "originalScore","1.0");
// Features we're extracting depend on external feature info not passed in
query.add("fl", "[fv]");
@ -120,13 +120,13 @@ public class TestExternalFeatures extends TestRerankBase {
// Adding efi in features section should make it work
query.remove("fl");
query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'");
// Adding efi in transformer + rq should still use the transformer's params for feature extraction
// Adding efi in transformer + rq should still returns features
query.remove("fl");
query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'");
}
@ -137,7 +137,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("rows", "1");
final String docs0fvalias_dense_csv = FeatureLoggerTestUtils.toFeatureVector(
"confidence","0.0",
"occurrences","0.0",
"originalScore","0.0");
final String docs0fvalias_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(
"originalScore","0.0");
@ -146,7 +146,7 @@ public class TestExternalFeatures extends TestRerankBase {
// Efi is explicitly not required, so we do not score the feature
query.remove("fl");
query.add("fl", "fvalias:[fv store=fstore2]");
query.add("fl", "fvalias:[fv store=fstore3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fvalias_default_csv+"'");
}