diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ba6e57492ac..4a8766edc27 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -195,6 +195,8 @@ Other Changes * SOLR-9914: SimpleFacets: refactor "contains" check into "SubstringBytesRefFilter" class. (Jonny Marks, David Smiley, Christine Poerschke) +* SOLR-10072: The test TestSelectiveWeightCreation appears to be unreliable. (Michael Nilsson via Mark Miller) + ================== 6.4.1 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json index 6c0cfa63452..d8a9ecae6f9 100644 --- a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json +++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json @@ -5,11 +5,11 @@ "q" : "{!terms f=title}${user_query}" } }, { - "name" : "confidence", - "class" : "org.apache.solr.ltr.feature.ValueFeature", + "name" : "docPopularity", + "class" : "org.apache.solr.ltr.feature.FieldValueFeature", "store": "fstore2", "params" : { - "value" : "${myconf}" + "field" : "popularity" } }, { "name":"originalScore", diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json index 04ab22968b0..68f659c3962 100644 --- a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json +++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json @@ -2,11 +2,11 @@ "class":"org.apache.solr.ltr.model.LinearModel", "name":"externalmodel", "features":[ - { "name": "matchedTitle"} + { "name": "titlePhraseMatch"} ], "params":{ "weights": { - "matchedTitle": 0.999 + "titlePhraseMatch": 0.555 } } } diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json similarity index 61% rename from solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json rename to solr/contrib/ltr/src/test-files/modelExamples/external_model2.json index f8e664855d3..f3d33f7faee 100644 --- a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json +++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json @@ -1,13 +1,13 @@ { "class":"org.apache.solr.ltr.model.LinearModel", - "name":"externalmodelstore", + "name":"externalmodel2", "store": "fstore2", "features":[ - { "name": "confidence"} + { "name": "docPopularity"} ], "params":{ "weights": { - "confidence": 0.999 + "docPopularity": 0.5 } } } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java index f4c21fd6607..630a68cf87f 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java @@ -25,21 +25,15 @@ public class TestParallelWeightCreation extends TestRerankBase{ public void testLTRScoringQueryParallelWeightCreationResultOrder() throws Exception { setuptest("solrconfig-ltr_Th10_10.xml", "schema.xml"); - assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", - "1")); - assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", - "2")); - assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", - "3")); - assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity", - "4")); - assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", - "5")); + assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", "1")); + assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2")); + assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3")); + assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4")); + assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5")); assertU(commit()); loadFeatures("external_features.json"); loadModels("external_model.json"); - loadModels("external_model_store.json"); // check to make sure that the order of results will be the same when using parallel weight creation final SolrQuery query = new SolrQuery(); @@ -47,10 +41,10 @@ public class TestParallelWeightCreation extends TestRerankBase{ query.add("fl", "*,score"); query.add("rows", "4"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); aftertest(); } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java index e44d4ac5496..5cfd999b360 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java @@ -106,21 +106,16 @@ public class TestSelectiveWeightCreation extends TestRerankBase { public static void before() throws Exception { setuptest(false); - assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", - "1")); - assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", - "2")); - assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", - "3")); - assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity", - "4")); - assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", - "5")); + assertU(adoc("id", "1", "title", "w3 w1", "description", "w1", "popularity", "1")); + assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2")); + assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3")); + assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4")); + assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5")); assertU(commit()); loadFeatures("external_features.json"); loadModels("external_model.json"); - loadModels("external_model_store.json"); + loadModels("external_model2.json"); } @AfterClass @@ -134,14 +129,14 @@ public class TestSelectiveWeightCreation extends TestRerankBase { final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newStringField("id", "0", Field.Store.YES)); + doc.add(newStringField("id", "10", Field.Store.YES)); doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc); doc = new Document(); - doc.add(newStringField("id", "1", Field.Store.YES)); + doc.add(newStringField("id", "11", Field.Store.YES)); // 1 extra token, but wizard and oz are close; doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); @@ -159,8 +154,8 @@ public class TestSelectiveWeightCreation extends TestRerankBase { // first run the standard query final TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(2, hits.totalHits); - assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); - assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); + assertEquals("10", searcher.doc(hits.scoreDocs[0].doc).get("id")); + assertEquals("11", searcher.doc(hits.scoreDocs[1].doc).get("id")); List features = makeFeatures(new int[] {0, 1, 2}); final List allFeatures = makeFeatures(new int[] {0, 1, 2, 3, 4, 5, @@ -206,7 +201,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase { } assertEquals(validFeatures, allFeatures.size()); - assertU(delI("0"));assertU(delI("1")); + assertU(delI("10"));assertU(delI("11")); r.close(); dir.close(); } @@ -216,41 +211,49 @@ public class TestSelectiveWeightCreation extends TestRerankBase { public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception { final String docs0fv = FeatureLoggerTestUtils.toFeatureVector( - "matchedTitle","1.0", "titlePhraseMatch","0.40254828"); + "matchedTitle","1.0", "titlePhraseMatch","0.6103343"); final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector( "popularity","3.0", "originalScore","1.0"); + // extract all features in externalmodel's store (default store) + // rerank using externalmodel (default store) final SolrQuery query = new SolrQuery(); query.setQuery("*:*"); - query.add("fl", "*,score"); - query.add("rows", "4"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - query.add("fl", "fv:[fv]"); + query.add("fl", "*,score,fv:[fv]"); + query.add("rows", "5"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); // extract all features in default store + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552"); + // extract all features from fstore4 + // rerank using externalmodel (default store) query.remove("fl"); query.remove("rq"); - query.add("fl", "*,score"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]"); - - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4 + query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552"); + // extract all features from fstore4 + // rerank using externalmodel2 (fstore2) query.remove("fl"); query.remove("rq"); - query.add("fl", "*,score"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodelstore efi.user_query=w3 efi.myconf=0.8}"); - query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); // score using fstore2 used by externalmodelstore - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7992"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4 + query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel2 efi.user_query=w3}"); + + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='5'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==2.5"); } } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java index 10ababb07f5..4010ee1900c 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java @@ -65,24 +65,24 @@ public class TestExternalFeatures extends TestRerankBase { assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==1.0"); - query.add("fl", "[fv]"); - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}"); + query.remove("fl"); + query.add("fl", "*,score,[fv]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0"); // Adding an efi in the transformer should not affect the rq ranking with a // different value for efi of the same parameter query.remove("fl"); - query.add("fl", "id,[fv efi.user_query=w2]"); + query.add("fl", "*,score,[fv efi.user_query=w2]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0"); } @Test @@ -92,7 +92,7 @@ public class TestExternalFeatures extends TestRerankBase { query.add("fl", "*,score,fv:[fv]"); query.add("rows", "1"); // Stopword only query passed in - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query='a'}"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a'}"); final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector( "matchedTitle","0.0", @@ -112,7 +112,7 @@ public class TestExternalFeatures extends TestRerankBase { query.add("rows", "1"); final String docs0fv_csv = FeatureLoggerTestUtils.toFeatureVector( - "confidence","2.3", "originalScore","1.0"); + "occurrences","2.3", "originalScore","1.0"); // Features we're extracting depend on external feature info not passed in query.add("fl", "[fv]"); @@ -120,13 +120,13 @@ public class TestExternalFeatures extends TestRerankBase { // Adding efi in features section should make it work query.remove("fl"); - query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]"); + query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'"); - // Adding efi in transformer + rq should still use the transformer's params for feature extraction + // Adding efi in transformer + rq should still returns features query.remove("fl"); - query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]"); - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}"); + query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'"); } @@ -137,7 +137,7 @@ public class TestExternalFeatures extends TestRerankBase { query.add("rows", "1"); final String docs0fvalias_dense_csv = FeatureLoggerTestUtils.toFeatureVector( - "confidence","0.0", + "occurrences","0.0", "originalScore","0.0"); final String docs0fvalias_sparse_csv = FeatureLoggerTestUtils.toFeatureVector( "originalScore","0.0"); @@ -146,7 +146,7 @@ public class TestExternalFeatures extends TestRerankBase { // Efi is explicitly not required, so we do not score the feature query.remove("fl"); - query.add("fl", "fvalias:[fv store=fstore2]"); + query.add("fl", "fvalias:[fv store=fstore3]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fvalias_default_csv+"'"); }