mirror of
https://github.com/discourse/discourse.git
synced 2025-02-19 01:46:01 +00:00
DEV: Adds RRF algorithm and API for adding results to search (#24202)
This commit is contained in:
Keegan George
GitHub
parent
e34d2cfde4
commit
eab9fbe277
@ -82,4 +82,6 @@
|
||||
</span>
|
||||
{{/if}}
|
||||
{{/if}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<PluginOutlet @name="after-search-result-entry" />
|
||||
@ -12,6 +12,7 @@ import {
|
||||
getSearchKey,
|
||||
isValidSearchTerm,
|
||||
logSearchLinkClick,
|
||||
reciprocallyRankedList,
|
||||
searchContextDescription,
|
||||
translateResults,
|
||||
updateRecentSearches,
|
||||
@ -78,6 +79,7 @@ export default Controller.extend({
|
||||
page: 1,
|
||||
resultCount: null,
|
||||
searchTypes: null,
|
||||
additionalSearchResults: [],
|
||||
selected: [],
|
||||
error: null,
|
||||
|
||||
@ -252,7 +254,7 @@ export default Controller.extend({
|
||||
return I18n.t("search.result_count", { count, plus, term });
|
||||
},
|
||||
|
||||
@observes("model.[posts,categories,tags,users].length")
|
||||
@observes("model.{posts,categories,tags,users}.length", "searchResultPosts")
|
||||
resultCountChanged() {
|
||||
if (!this.model.posts) {
|
||||
return 0;
|
||||
@ -260,7 +262,7 @@ export default Controller.extend({
|
||||
|
||||
this.set(
|
||||
"resultCount",
|
||||
this.model.posts.length +
|
||||
this.searchResultPosts.length +
|
||||
this.model.categories.length +
|
||||
this.model.tags.length +
|
||||
this.model.users.length
|
||||
@ -274,7 +276,7 @@ export default Controller.extend({
|
||||
|
||||
hasSelection: gt("selected.length", 0),
|
||||
|
||||
@discourseComputed("selected.length", "model.posts.length")
|
||||
@discourseComputed("selected.length", "searchResultPosts.length")
|
||||
hasUnselectedResults(selectionCount, postsCount) {
|
||||
return selectionCount < postsCount;
|
||||
},
|
||||
@ -308,6 +310,18 @@ export default Controller.extend({
|
||||
: "search-info";
|
||||
},
|
||||
|
||||
@discourseComputed("model.posts", "additionalSearchResults")
|
||||
searchResultPosts(posts, additionalSearchResults) {
|
||||
if (additionalSearchResults?.list?.length > 0) {
|
||||
return reciprocallyRankedList(
|
||||
[posts, additionalSearchResults.list],
|
||||
["topic_id", additionalSearchResults.identifier]
|
||||
);
|
||||
} else {
|
||||
return posts;
|
||||
}
|
||||
},
|
||||
|
||||
searchButtonDisabled: or("searching", "loading"),
|
||||
|
||||
@bind
|
||||
@ -464,9 +478,17 @@ export default Controller.extend({
|
||||
});
|
||||
},
|
||||
|
||||
@action
|
||||
addSearchResults(list, identifier) {
|
||||
this.set("additionalSearchResults", {
|
||||
list,
|
||||
identifier,
|
||||
});
|
||||
},
|
||||
|
||||
actions: {
|
||||
selectAll() {
|
||||
this.selected.addObjects(this.get("model.posts").mapBy("topic"));
|
||||
this.selected.addObjects(this.get("searchResultPosts").mapBy("topic"));
|
||||
|
||||
// Doing this the proper way is a HUGE pain,
|
||||
// we can hack this to work by observing each on the array
|
||||
|
||||
@ -259,3 +259,73 @@ export function logSearchLinkClick(params) {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* reciprocallyRankedList() makes use of the Reciprocal Ranking Fusion Algorithm (RRF)
|
||||
*
|
||||
* A method used to combine rankings from multiple sources
|
||||
* to aggregate them to provide a single improved ranking
|
||||
*
|
||||
* RRF = 1 / k + r(d)
|
||||
*
|
||||
* k = a constant, small positive value to avoid division by zero
|
||||
* r(d) = the reciprocal rank of the item in the ranking list
|
||||
*
|
||||
*
|
||||
* @param {Array} lists - an array of arrays containing the results from each source
|
||||
* The passed-in list must include the properties specified in the `identifiers` array
|
||||
* @param {Array} identifiers - an array of property names used to identify items in the ranking lists
|
||||
*
|
||||
* Example Usage: reciprocallyRankedList([list1, list2, list3], ["id", "topic_id", "uuid"])
|
||||
*
|
||||
**/
|
||||
export function reciprocallyRankedList(lists, identifiers) {
|
||||
const k = 5;
|
||||
|
||||
if (lists.length === 1) {
|
||||
return lists;
|
||||
}
|
||||
|
||||
if (lists.length !== identifiers.length) {
|
||||
throw new Error("The number of lists must match the number of identifiers");
|
||||
}
|
||||
|
||||
if (lists.length === 0) {
|
||||
throw new Error("Lists must not be an empty array");
|
||||
}
|
||||
|
||||
// Assign a reciprocal rank to each result
|
||||
lists.forEach((list) => {
|
||||
list.forEach((listItem, index) => {
|
||||
const identifierValues = identifiers.map((id) => listItem[id]);
|
||||
const itemKey = identifierValues.join("_");
|
||||
listItem.reciprocalRank = 1 / (index + k);
|
||||
listItem.itemKey = itemKey;
|
||||
});
|
||||
});
|
||||
|
||||
// Combine lists into a single list
|
||||
const combinedList = [].concat(...lists);
|
||||
|
||||
// Remove duplicates and sum reciprocal ranks based on identifiers
|
||||
const resultMap = new Map();
|
||||
combinedList.forEach((result) => {
|
||||
const existingResult = resultMap.get(result.itemKey);
|
||||
if (!existingResult) {
|
||||
resultMap.set(result.itemKey, result);
|
||||
} else {
|
||||
// Sum reciprocal ranks for duplicates
|
||||
existingResult.reciprocalRank += result.reciprocalRank;
|
||||
}
|
||||
});
|
||||
|
||||
// Convert the map values back to an array
|
||||
const uniqueResults = Array.from(resultMap.values());
|
||||
|
||||
// Sort the results by reciprocal ranking
|
||||
const sortedResults = uniqueResults.sort(
|
||||
(a, b) => b.reciprocalRank - a.reciprocalRank
|
||||
);
|
||||
|
||||
return sortedResults;
|
||||
}
|
||||
|
||||
@ -86,7 +86,12 @@
|
||||
<PluginOutlet
|
||||
@name="full-page-search-below-search-header"
|
||||
@connectorTagName="div"
|
||||
@outletArgs={{hash search=this.searchTerm type=this.search_type}}
|
||||
@outletArgs={{hash
|
||||
search=this.searchTerm
|
||||
type=this.search_type
|
||||
model=this.model
|
||||
addSearchResults=this.addSearchResults
|
||||
}}
|
||||
/>
|
||||
|
||||
{{#if this.hasResults}}
|
||||
@ -164,7 +169,7 @@
|
||||
<LoadMore @selector=".fps-result" @action={{action "loadMore"}}>
|
||||
{{#if (or this.usingDefaultSearchType this.customSearchType)}}
|
||||
<SearchResultEntries
|
||||
@posts={{this.model.posts}}
|
||||
@posts={{this.searchResultPosts}}
|
||||
@bulkSelectEnabled={{this.bulkSelectEnabled}}
|
||||
@selected={{this.selected}}
|
||||
@highlightQuery={{this.highlightQuery}}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { setupTest } from "ember-qunit";
|
||||
import { module, test } from "qunit";
|
||||
import {
|
||||
reciprocallyRankedList,
|
||||
searchContextDescription,
|
||||
translateResults,
|
||||
} from "discourse/lib/search";
|
||||
@ -62,4 +63,505 @@ module("Unit | Utility | search", function (hooks) {
|
||||
);
|
||||
assert.strictEqual(searchContextDescription("bad_type"), undefined);
|
||||
});
|
||||
|
||||
test("reciprocallyRankedList", async function (assert) {
|
||||
const sourceA = [
|
||||
{
|
||||
id: 250,
|
||||
name: "Bruce Wayne",
|
||||
username: "batman",
|
||||
topic_id: 96,
|
||||
topic: {
|
||||
id: 96,
|
||||
title: "I like to fight crime",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 104,
|
||||
name: "Steve Rogers",
|
||||
username: "captain_america",
|
||||
topic_id: 2,
|
||||
topic: {
|
||||
id: 2,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 202,
|
||||
name: "Peter Parker",
|
||||
username: "spidey",
|
||||
topic_id: 32,
|
||||
topic: {
|
||||
id: 32,
|
||||
title: "My experience meeting the Avengers",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 290,
|
||||
name: "Clark Kent",
|
||||
username: "superman",
|
||||
topic_id: 111,
|
||||
topic: {
|
||||
id: 111,
|
||||
title: "My fear of Kryptonite",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const sourceB = [
|
||||
{
|
||||
id: 104,
|
||||
name: "Tony Stark",
|
||||
username: "ironman",
|
||||
topic_id: 95,
|
||||
topic: {
|
||||
id: 95,
|
||||
title: "What I learned from my father",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 246,
|
||||
name: "The Joker",
|
||||
username: "joker",
|
||||
topic_id: 93,
|
||||
topic: {
|
||||
id: 93,
|
||||
title: "Why don't you put a smile on that face...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 104,
|
||||
name: "Steve Rogers",
|
||||
username: "captain_america",
|
||||
topic_id: 2,
|
||||
topic: {
|
||||
id: 2,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 245,
|
||||
name: "Loki",
|
||||
username: "loki",
|
||||
topic_id: 92,
|
||||
topic: {
|
||||
id: 92,
|
||||
title: "There is only one person you can trust",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const desiredMixedResults = [
|
||||
{
|
||||
id: 104,
|
||||
itemKey: "2_2",
|
||||
name: "Steve Rogers",
|
||||
reciprocalRank: 0.30952380952380953,
|
||||
topic: {
|
||||
id: 2,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
topic_id: 2,
|
||||
username: "captain_america",
|
||||
},
|
||||
{
|
||||
id: 250,
|
||||
itemKey: "96_96",
|
||||
name: "Bruce Wayne",
|
||||
reciprocalRank: 0.2,
|
||||
topic: {
|
||||
id: 96,
|
||||
title: "I like to fight crime",
|
||||
},
|
||||
topic_id: 96,
|
||||
username: "batman",
|
||||
},
|
||||
{
|
||||
id: 104,
|
||||
itemKey: "95_95",
|
||||
name: "Tony Stark",
|
||||
reciprocalRank: 0.2,
|
||||
topic: {
|
||||
id: 95,
|
||||
title: "What I learned from my father",
|
||||
},
|
||||
topic_id: 95,
|
||||
username: "ironman",
|
||||
},
|
||||
{
|
||||
id: 246,
|
||||
itemKey: "93_93",
|
||||
name: "The Joker",
|
||||
reciprocalRank: 0.16666666666666666,
|
||||
topic: {
|
||||
id: 93,
|
||||
title: "Why don't you put a smile on that face...",
|
||||
},
|
||||
topic_id: 93,
|
||||
username: "joker",
|
||||
},
|
||||
{
|
||||
id: 202,
|
||||
itemKey: "32_32",
|
||||
name: "Peter Parker",
|
||||
reciprocalRank: 0.14285714285714285,
|
||||
topic: {
|
||||
id: 32,
|
||||
title: "My experience meeting the Avengers",
|
||||
},
|
||||
topic_id: 32,
|
||||
username: "spidey",
|
||||
},
|
||||
{
|
||||
id: 290,
|
||||
itemKey: "111_111",
|
||||
name: "Clark Kent",
|
||||
reciprocalRank: 0.125,
|
||||
topic: {
|
||||
id: 111,
|
||||
title: "My fear of Kryptonite",
|
||||
},
|
||||
topic_id: 111,
|
||||
username: "superman",
|
||||
},
|
||||
{
|
||||
id: 245,
|
||||
itemKey: "92_92",
|
||||
name: "Loki",
|
||||
reciprocalRank: 0.125,
|
||||
topic: {
|
||||
id: 92,
|
||||
title: "There is only one person you can trust",
|
||||
},
|
||||
topic_id: 92,
|
||||
username: "loki",
|
||||
},
|
||||
];
|
||||
|
||||
const rankedList = reciprocallyRankedList(
|
||||
[sourceA, sourceB],
|
||||
["topic_id", "topic_id"]
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
rankedList,
|
||||
desiredMixedResults,
|
||||
"it correctly ranks the results using the reciprocal ranking algorithm"
|
||||
);
|
||||
});
|
||||
|
||||
test("reciprocallyRankedList (varied lists with more sources)", async function (assert) {
|
||||
const sourceA = [
|
||||
{
|
||||
id: 1,
|
||||
name: "Tony Stark",
|
||||
username: "ironman",
|
||||
topic_id: 21,
|
||||
topic: {
|
||||
id: 21,
|
||||
title: "I am iron man",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
name: "Steve Rogers",
|
||||
username: "captain_america",
|
||||
topic_id: 22,
|
||||
topic: {
|
||||
id: 22,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: "Peter Parker",
|
||||
username: "spidey",
|
||||
topic_id: 23,
|
||||
topic: {
|
||||
id: 23,
|
||||
title: "My experience meeting the Avengers",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
name: "Stephen Strange",
|
||||
username: "doctor_strange",
|
||||
topic_id: 24,
|
||||
topic: {
|
||||
id: 24,
|
||||
title: "14 mil different possible futures",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const sourceB = [
|
||||
{
|
||||
id: 5,
|
||||
name: "Clark Kent",
|
||||
username: "superman",
|
||||
tid: 90,
|
||||
topic: {
|
||||
id: 90,
|
||||
title: "I am not from this planet.",
|
||||
fancy_title: "I am not from this planet.",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 6,
|
||||
name: "Bruce Wayne",
|
||||
username: "batman",
|
||||
tid: 91,
|
||||
topic: {
|
||||
id: 91,
|
||||
title: "It's not who I am underneath, but what I do that defines me.",
|
||||
fancy_title: "It's what I do that defines me.",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 7,
|
||||
name: "Steve Rogers",
|
||||
username: "captain_america",
|
||||
tid: 22,
|
||||
topic: {
|
||||
id: 22,
|
||||
title: "What its like being frozen...",
|
||||
fancy_title: "What its like being frozen...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 8,
|
||||
name: "Barry Allen",
|
||||
username: "the_flash",
|
||||
tid: 93,
|
||||
topic: {
|
||||
id: 93,
|
||||
title: "Run Barry run!",
|
||||
fancy_title: "Run barry run!",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const sourceC = [
|
||||
{
|
||||
id: 41,
|
||||
tuid: 906,
|
||||
name: "The Joker",
|
||||
username: "joker",
|
||||
user_id: 81,
|
||||
flair_name: "DC",
|
||||
topic: {
|
||||
title: "I am not from this planet.",
|
||||
can_edit: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 91,
|
||||
tuid: 23,
|
||||
name: "Peter Parker",
|
||||
username: "spidey",
|
||||
user_id: 80,
|
||||
flair_name: "Marvel",
|
||||
topic: {
|
||||
title: "My experience meeting the Avengers.",
|
||||
can_edit: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 42,
|
||||
tuid: 96,
|
||||
name: "Thanos",
|
||||
username: "thanos",
|
||||
user_id: 82,
|
||||
flair_name: "Marvel",
|
||||
topic: {
|
||||
title: "Fine, I'll do it myself",
|
||||
can_edit: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 43,
|
||||
tuid: 97,
|
||||
name: "Lex Luthor",
|
||||
username: "lex",
|
||||
user_id: 83,
|
||||
flair_name: "DC",
|
||||
topic: {
|
||||
title:
|
||||
"Devils don't come from the hell beneath us, they come from the sky",
|
||||
can_edit: true,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const desiredMixedResults = [
|
||||
{
|
||||
id: 1,
|
||||
itemKey: "21__",
|
||||
name: "Tony Stark",
|
||||
reciprocalRank: 0.2,
|
||||
topic: {
|
||||
id: 21,
|
||||
title: "I am iron man",
|
||||
},
|
||||
topic_id: 21,
|
||||
username: "ironman",
|
||||
},
|
||||
{
|
||||
id: 5,
|
||||
itemKey: "_90_",
|
||||
name: "Clark Kent",
|
||||
reciprocalRank: 0.2,
|
||||
tid: 90,
|
||||
topic: {
|
||||
fancy_title: "I am not from this planet.",
|
||||
id: 90,
|
||||
title: "I am not from this planet.",
|
||||
},
|
||||
username: "superman",
|
||||
},
|
||||
{
|
||||
flair_name: "DC",
|
||||
id: 41,
|
||||
itemKey: "__906",
|
||||
name: "The Joker",
|
||||
reciprocalRank: 0.2,
|
||||
topic: {
|
||||
can_edit: true,
|
||||
title: "I am not from this planet.",
|
||||
},
|
||||
tuid: 906,
|
||||
user_id: 81,
|
||||
username: "joker",
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
itemKey: "22__",
|
||||
name: "Steve Rogers",
|
||||
reciprocalRank: 0.16666666666666666,
|
||||
topic: {
|
||||
id: 22,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
topic_id: 22,
|
||||
username: "captain_america",
|
||||
},
|
||||
{
|
||||
id: 6,
|
||||
itemKey: "_91_",
|
||||
name: "Bruce Wayne",
|
||||
reciprocalRank: 0.16666666666666666,
|
||||
tid: 91,
|
||||
topic: {
|
||||
fancy_title: "It's what I do that defines me.",
|
||||
id: 91,
|
||||
title: "It's not who I am underneath, but what I do that defines me.",
|
||||
},
|
||||
username: "batman",
|
||||
},
|
||||
{
|
||||
flair_name: "Marvel",
|
||||
id: 91,
|
||||
itemKey: "__23",
|
||||
name: "Peter Parker",
|
||||
reciprocalRank: 0.16666666666666666,
|
||||
topic: {
|
||||
can_edit: false,
|
||||
title: "My experience meeting the Avengers.",
|
||||
},
|
||||
tuid: 23,
|
||||
user_id: 80,
|
||||
username: "spidey",
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
itemKey: "23__",
|
||||
name: "Peter Parker",
|
||||
reciprocalRank: 0.14285714285714285,
|
||||
topic: {
|
||||
id: 23,
|
||||
title: "My experience meeting the Avengers",
|
||||
},
|
||||
topic_id: 23,
|
||||
username: "spidey",
|
||||
},
|
||||
{
|
||||
id: 7,
|
||||
itemKey: "_22_",
|
||||
name: "Steve Rogers",
|
||||
reciprocalRank: 0.14285714285714285,
|
||||
tid: 22,
|
||||
topic: {
|
||||
fancy_title: "What its like being frozen...",
|
||||
id: 22,
|
||||
title: "What its like being frozen...",
|
||||
},
|
||||
username: "captain_america",
|
||||
},
|
||||
{
|
||||
flair_name: "Marvel",
|
||||
id: 42,
|
||||
itemKey: "__96",
|
||||
name: "Thanos",
|
||||
reciprocalRank: 0.14285714285714285,
|
||||
topic: {
|
||||
can_edit: true,
|
||||
title: "Fine, I'll do it myself",
|
||||
},
|
||||
tuid: 96,
|
||||
user_id: 82,
|
||||
username: "thanos",
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
itemKey: "24__",
|
||||
name: "Stephen Strange",
|
||||
reciprocalRank: 0.125,
|
||||
topic: {
|
||||
id: 24,
|
||||
title: "14 mil different possible futures",
|
||||
},
|
||||
topic_id: 24,
|
||||
username: "doctor_strange",
|
||||
},
|
||||
{
|
||||
id: 8,
|
||||
itemKey: "_93_",
|
||||
name: "Barry Allen",
|
||||
reciprocalRank: 0.125,
|
||||
tid: 93,
|
||||
topic: {
|
||||
fancy_title: "Run barry run!",
|
||||
id: 93,
|
||||
title: "Run Barry run!",
|
||||
},
|
||||
username: "the_flash",
|
||||
},
|
||||
{
|
||||
flair_name: "DC",
|
||||
id: 43,
|
||||
itemKey: "__97",
|
||||
name: "Lex Luthor",
|
||||
reciprocalRank: 0.125,
|
||||
topic: {
|
||||
can_edit: true,
|
||||
title:
|
||||
"Devils don't come from the hell beneath us, they come from the sky",
|
||||
},
|
||||
tuid: 97,
|
||||
user_id: 83,
|
||||
username: "lex",
|
||||
},
|
||||
];
|
||||
|
||||
const rankedList = reciprocallyRankedList(
|
||||
[sourceA, sourceB, sourceC],
|
||||
["topic_id", "tid", "tuid"]
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
rankedList,
|
||||
desiredMixedResults,
|
||||
"it correctly ranks the results using the reciprocal ranking algorithm"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user