FEATURE: Render Related Topics for Crawlers (#386)

app/views/list/related_topics.html.erb

@@ -0,0 +1,71 @@
+<div id="related-topics" class="more-topics__list " role="complementary" aria-labelledby="related-topics-title">
+  <h3 id="related-topics-title" class="more-topics__list-title">
+    <%= t 'js.discourse_ai.related_topics.title' %>
+  </h3>
+  <div class="topic-list-container" itemscope itemtype='http://schema.org/ItemList'>
+    <meta itemprop='itemListOrder' content='http://schema.org/ItemListOrderDescending'>
+    <table class='topic-list'>
+      <thead>
+        <tr>
+          <th><%= t 'js.topic.title' %></th>
+          <th></th>
+          <th class="replies"><%= t 'js.replies' %></th>
+          <th class="views"><%= t 'js.views' %></th>
+          <th><%= t 'js.activity' %></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @list.each_with_index do |t,i| %>
+          <tr class="topic-list-item">
+            <td class="main-link" itemprop='itemListElement' itemscope itemtype='http://schema.org/ListItem'>
+              <meta itemprop='position' content='<%= i + 1 %>'>
+              <span class="link-top-line">
+                <a itemprop='url' href='<%= t.url %>' class='title raw-link raw-topic-link'><%= t.title %></a>
+              </span>
+              <div class="link-bottom-line">
+                <% if (!@category || @category.has_children?) && t.category && !t.category.uncategorized? %>
+                  <a href='<%= t.category.url %>' class='badge-wrapper bullet'>
+                    <span class='badge-category-bg' style='background-color: #<%= t.category.color %>'></span>
+                    <span class='badge-category clear-badge'>
+                      <span class='category-name'><%= t.category.name %></span>
+                    </span>
+                  </a>
+                <% end %>
+                <% if tags = t.visible_tags(guardian) %>
+                  <div class="discourse-tags">
+                    <% tags.each_with_index do |tag, index| %>
+                      <a href='<%= tag.full_url %>' class='discourse-tag'><%= tag.name %></a>
+                      <% if index < tags.size - 1 %>,&nbsp;<% end %>
+                    <% end %>
+                  </div>
+                <% end %>
+              </div>
+              <% if t.pinned_until && (t.pinned_until > Time.zone.now) && (t.pinned_globally || @list.category) && t.excerpt %>
+                <p class='excerpt'>
+                  <%= t.excerpt.html_safe %>
+                </p>
+              <% end %>
+            </td>
+            <td class='posters'>
+              <% t.posters.each do |poster| %>
+                <a href="<%= Discourse.base_url %>/u/<%= poster.user.username %>" class="<%= poster.extras %>">
+                  <%- poster_name_and_description = h(poster.name_and_description) %>
+                  <img width="25" height="25" src="<%= poster.user.avatar_template.gsub('{size}', '25') %>" class="avatar" title='<%= poster_name_and_description %>' aria-label='<%= poster_name_and_description %>'>
+                </a>
+              <% end %>
+            </td>
+            <td class="replies">
+              <span class='posts' title='<%= t 'posts' %>'><%= t.posts_count - 1 %></span>
+            </td>
+            <td class="views">
+              <span class='views' title='<%= t 'views' %>'><%= t.views %></span>
+            </td>
+            <td>
+              <%= I18n.l(t.last_posted_at || t.created_at, format: :date_only) %>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+</div>

lib/embeddings/entry_point.rb

@@ -37,6 +37,10 @@ module DiscourseAi
           end
         end
 
+        plugin.register_html_builder("server:topic-show-after-posts-crawler") do |controller|
+          SemanticRelated.related_topics_for_crawler(controller)
+        end
+
         # embeddings generation.
         callback =
           Proc.new do |topic|

lib/embeddings/semantic_related.rb

@@ -58,6 +58,30 @@ module DiscourseAi
         end
       end
 
+      def self.related_topics_for_crawler(controller)
+        return "" if !controller.instance_of? TopicsController
+        return "" if !SiteSetting.ai_embeddings_semantic_related_topics_enabled
+        return "" if SiteSetting.ai_embeddings_semantic_related_topics < 1
+
+        topic_view = controller.instance_variable_get(:@topic_view)
+        topic = topic_view&.topic
+        return "" if !topic
+
+        related_topics ||= SemanticTopicQuery.new(nil).list_semantic_related_topics(topic).topics
+
+        return "" if related_topics.empty?
+
+        render_result =
+          ApplicationController.render(
+            template: "list/related_topics",
+            layout: false,
+            assigns: {
+              list: related_topics,
+              topic: topic,
+            },
+          )
+      end
+
       private
 
       def semantic_suggested_key(topic_id)