FIX: Load categories from search response (#612)

When lazy load categories is enabled, the list of categories does not
have to fetched from the "site.json" endpoint because it is already
returned by "search.json".

This commit reverts commits 5056502 and 3e54697 because iterating over
all pages of categories is not really necessary.

lib/ai_bot/tools/discourse_meta_search.rb

@@ -110,7 +110,13 @@ module DiscourseAi
           if posts.blank?
             { args: parameters, rows: [], instruction: "nothing was found, expand your search" }
           else
-            categories = self.class.categories
+            categories =
+              if categories_json = json.dig("grouped_search_result", "extra", "categories")
+                categories_json.map { |c| [c["id"], c] }.to_h
+              else
+                self.class.categories
+              end
+
             topics = (json["topics"]).map { |t| [t["id"], t] }.to_h
 
             format_results(posts, args: parameters) do |post|
@@ -148,25 +154,14 @@ module DiscourseAi
         def self.categories
           return @categories if defined?(@categories)
 
-          @categories = {}
+          url = "https://meta.discourse.org/site.json"
-
+          json = JSON.parse(Net::HTTP.get(URI(url)))
-          page = 0
+          @categories =
-          loop do
+            json["categories"]
-            page += 1
+              .map do |c|
-            url = "https://meta.discourse.org/categories.json?page=#{page}"
+                [c["id"], { "name" => c["name"], "parent_category_id" => c["parent_category_id"] }]
-
+              end
-            json = JSON.parse(Net::HTTP.get(URI(url)))
+              .to_h
-            break if json["category_list"]["categories"].blank?
-
-            json["category_list"]["categories"].each do |c|
-              @categories[c["id"]] = {
-                "name" => c["name"],
-                "parent_category_id" => c["parent_category_id"],
-              }
-            end
-          end
-
-          @categories
         end
 
         def description_args

spec/fixtures/search_meta/categories_page_2.json

@@ -1,7 +0,0 @@
-{
-  "category_list": {
-    "can_create_category": false,
-    "can_create_topic": false,
-    "categories": []
-  }
-}

spec/lib/modules/ai_bot/tools/discourse_meta_search_spec.rb

@@ -11,27 +11,16 @@ RSpec.describe DiscourseAi::AiBot::Tools::DiscourseMetaSearch do
 
   let(:mock_search_json) { plugin_file_from_fixtures("search.json", "search_meta").read }
 
-  let(:mock_categories_page_1) do
+  let(:mock_search_with_categories_json) do
-    plugin_file_from_fixtures("categories_page_1.json", "search_meta").read
+    plugin_file_from_fixtures("search_with_categories.json", "search_meta").read
   end
 
-  let(:mock_categories_page_2) do
+  let(:mock_site_json) { plugin_file_from_fixtures("site.json", "search_meta").read }
-    plugin_file_from_fixtures("categories_page_2.json", "search_meta").read
-  end
 
   before do
-    stub_request(:get, "https://meta.discourse.org/categories.json?page=1").to_return(
+    stub_request(:get, "https://meta.discourse.org/site.json").to_return(
       status: 200,
-      body: mock_categories_page_1,
+      body: mock_site_json,
-      headers: {
-      },
-    )
-  end
-
-  before do
-    stub_request(:get, "https://meta.discourse.org/categories.json?page=2").to_return(
-      status: 200,
-      body: mock_categories_page_2,
       headers: {
       },
     )
@@ -54,6 +43,23 @@ RSpec.describe DiscourseAi::AiBot::Tools::DiscourseMetaSearch do
     )
   end
 
+  it "searches meta.discourse.org with lazy_load_categories enabled" do
+    stub_request(:get, "https://meta.discourse.org/search.json?q=test").to_return(
+      status: 200,
+      body: mock_search_with_categories_json,
+      headers: {
+      },
+    )
+
+    search = described_class.new({ search_query: "test" }, bot_user: bot_user, llm: llm)
+    results = search.invoke(&progress_blk)
+    expect(results[:rows].length).to eq(20)
+
+    expect(results[:rows].first[results[:column_names].index("category")]).to eq(
+      "documentation > developers",
+    )
+  end
+
   it "passes on all search parameters" do
     url =
       "https://meta.discourse.org/search.json?q=test%20category:test%20user:test%20order:test%20max_posts:1%20tags:test%20before:test%20after:test%20status:test"