FEATURE: Append limited search results with semantic search (#35446)

## 🔍 Overview This update adds AI enhancement to the quick search feature of Discourse. When the setting: `ai_embeddings_semantic_quick_search_enabled` is enabled and a search is made using quick search menu yielding only a few results or none, AI results will be appended to the result. ## 📹 Preview ### Before https://github.com/user-attachments/assets/f3880cc3-9e9e-4b1e-8fd2-bb2cc27fd7de ### After https://github.com/user-attachments/assets/c6975690-2aa3-44a9-b8e8-ad4d42effa9b
2026-08-02 09:29:37 -05:00 · 2025-10-16 09:27:31 -07:00
parent bd94fcbce6
commit 5dd149079b
7 changed files with 308 additions and 148 deletions
@@ -68,9 +68,6 @@ module DiscourseAi
      end

      def quick_search
-        # this search function searches posts (vs: topics)
-        # it requires post embeddings and a reranker
-        # it will not perform a hyde expantion
        query = params[:q].to_s

        if query.length < SiteSetting.min_search_term_length
@@ -83,6 +80,7 @@ module DiscourseAi
            term: query,
            search_context: guardian,
            use_pg_headlines_for_excerpt: false,
+            can_lazy_load_categories: guardian.can_lazy_load_categories?,
          )

        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
@@ -92,7 +90,9 @@ module DiscourseAi
        end

        hijack do
-          semantic_search.quick_search(query).each { |topic_post| grouped_results.add(topic_post) }
+          semantic_search
+            .search_for_topics(query, _page = 1, hyde: false)
+            .each { |topic_post| grouped_results.add(topic_post) }

          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
        end
@@ -1,76 +0,0 @@
-import Component from "@glimmer/component";
-import { action } from "@ember/object";
-import { service } from "@ember/service";
-import AssistantItem from "discourse/components/search-menu/results/assistant-item";
-import { ajax } from "discourse/lib/ajax";
-import { popupAjaxError } from "discourse/lib/ajax-error";
-import { isValidSearchTerm, translateResults } from "discourse/lib/search";
-import { i18n } from "discourse-i18n";
-
-export default class AiQuickSemanticSearch extends Component {
-  static shouldRender(args, { siteSettings }) {
-    return siteSettings.ai_embeddings_semantic_quick_search_enabled;
-  }
-
-  @service search;
-  @service quickSearch;
-  @service siteSettings;
-
-  @action
-  async searchTermChanged() {
-    if (!this.search.activeGlobalSearchTerm) {
-      this.search.noResults = false;
-      this.search.results = {};
-      this.quickSearch.loading = false;
-      this.quickSearch.invalidTerm = false;
-    } else if (
-      !isValidSearchTerm(this.search.activeGlobalSearchTerm, this.siteSettings)
-    ) {
-      this.search.noResults = true;
-      this.search.results = {};
-      this.quickSearch.loading = false;
-      this.quickSearch.invalidTerm = true;
-      return;
-    } else {
-      await this.performSearch();
-    }
-  }
-
-  async performSearch() {
-    this.quickSearch.loading = true;
-    this.quickSearch.invalidTerm = false;
-
-    try {
-      const results = await ajax(`/discourse-ai/embeddings/quick-search`, {
-        data: {
-          q: this.search.activeGlobalSearchTerm,
-        },
-      });
-
-      const searchResults = await translateResults(results);
-
-      if (searchResults) {
-        this.search.noResults = results.resultTypes.length === 0;
-        this.search.results = searchResults;
-      }
-    } catch (error) {
-      popupAjaxError(error);
-    } finally {
-      this.quickSearch.loading = false;
-    }
-  }
-
-  <template>
-    {{yield}}
-
-    {{#if this.search.activeGlobalSearchTerm}}
-      <AssistantItem
-        @suffix={{i18n "discourse_ai.embeddings.quick_search.suffix"}}
-        @icon="discourse-sparkles"
-        @closeSearchMenu={{@closeSearchMenu}}
-        @searchTermChanged={{this.searchTermChanged}}
-        @suggestionKeyword={{@suggestionKeyword}}
-      />
-    {{/if}}
-  </template>
-}
@@ -0,0 +1,198 @@
+import Component from "@glimmer/component";
+import { tracked } from "@glimmer/tracking";
+import { action } from "@ember/object";
+import didUpdate from "@ember/render-modifiers/modifiers/did-update";
+import { service } from "@ember/service";
+import { modifier } from "ember-modifier";
+import { MODIFIER_REGEXP } from "discourse/components/search-menu";
+import loadingSpinner from "discourse/helpers/loading-spinner";
+import { ajax } from "discourse/lib/ajax";
+import { popupAjaxError } from "discourse/lib/ajax-error";
+import { isValidSearchTerm, translateResults } from "discourse/lib/search";
+
+const MAX_RESULTS_FOR_ADDING_AI = 3;
+
+export default class AiQuickSearch extends Component {
+  static shouldRender(args, { siteSettings }) {
+    return siteSettings.ai_embeddings_semantic_quick_search_enabled;
+  }
+
+  @service search;
+  @service siteSettings;
+
+  @tracked hasAiResults = false;
+  @tracked searchingWithAi = false;
+  @tracked lastSearchTerm = null;
+
+  markAiResults = modifier(() => {
+    if (!this.hasAiResults) {
+      return;
+    }
+
+    const resultsContainer = document.querySelector(".search-menu .results");
+    if (resultsContainer) {
+      resultsContainer.classList.add("has-ai-search-results");
+    }
+
+    const resultTypes = this.search.results?.resultTypes || [];
+    resultTypes.forEach((resultType) => {
+      resultType.results?.forEach((result) => {
+        if (result.aiGenerated) {
+          const topicId = result.topic?.id || result.topic_id;
+          if (topicId) {
+            const listItem = document
+              .querySelector(
+                `.search-menu .list .item [data-topic-id="${topicId}"]`
+              )
+              ?.closest(".item");
+
+            if (listItem) {
+              listItem.classList.add("ai-search-result");
+            }
+          }
+        }
+      });
+    });
+  });
+
+  get totalResults() {
+    const resultTypes = this.search.results?.resultTypes || [];
+    return resultTypes.reduce(
+      (sum, type) => sum + (type.results?.length || 0),
+      0
+    );
+  }
+
+  get hasModifiers() {
+    const term = this.search.activeGlobalSearchTerm || "";
+    return MODIFIER_REGEXP.test(term);
+  }
+
+  get shouldAddAiResults() {
+    const searchingForTopics = this.args.outletArgs?.searchTopics;
+    const hasSearched = this.search.results !== undefined;
+
+    return (
+      !this.hasAiResults &&
+      !this.searchingWithAi &&
+      searchingForTopics &&
+      hasSearched &&
+      isValidSearchTerm(
+        this.search.activeGlobalSearchTerm,
+        this.siteSettings
+      ) &&
+      this.totalResults <= MAX_RESULTS_FOR_ADDING_AI &&
+      !this.hasModifiers &&
+      !this.search.inTopicContext
+    );
+  }
+
+  @action
+  onSearchTermChange() {
+    this.hasAiResults = false;
+    this.searchingWithAi = false;
+    this.lastSearchTerm = null;
+
+    const resultsContainer = document.querySelector(".search-menu .results");
+    if (resultsContainer) {
+      resultsContainer.classList.remove("has-ai-search-results");
+    }
+  }
+
+  @action
+  async checkAndAddAiResults() {
+    const currentTerm = this.search.activeGlobalSearchTerm;
+    if (this.lastSearchTerm === currentTerm) {
+      return;
+    }
+
+    if (this.shouldAddAiResults) {
+      this.lastSearchTerm = currentTerm;
+      await this.performAiSearch();
+    }
+  }
+
+  async performAiSearch() {
+    this.searchingWithAi = true;
+
+    if (this.totalResults === 0) {
+      this.search.noResults = false;
+    }
+
+    try {
+      const results = await ajax("/discourse-ai/embeddings/quick-search", {
+        data: {
+          q: this.search.activeGlobalSearchTerm,
+        },
+      });
+
+      const searchResults = await translateResults(results);
+
+      if (searchResults?.posts?.length > 0) {
+        searchResults.posts.forEach((post) => {
+          post.aiGenerated = true;
+        });
+
+        this.appendResults(searchResults);
+        this.hasAiResults = true;
+      } else {
+        this.search.noResults = true;
+      }
+    } catch (error) {
+      popupAjaxError(error);
+      if (this.totalResults === 0) {
+        this.search.noResults = true;
+      }
+    } finally {
+      this.searchingWithAi = false;
+    }
+  }
+
+  appendResults(aiResults) {
+    if (!this.search.results) {
+      this.search.results = {};
+    }
+
+    this.search.results.posts = [
+      ...(this.search.results.posts || []),
+      ...aiResults.posts,
+    ];
+
+    const resultTypes = this.search.results.resultTypes || [];
+    const topicResultType = resultTypes.find((rt) => rt.type === "topic");
+
+    if (topicResultType) {
+      topicResultType.results = [
+        ...topicResultType.results,
+        ...aiResults.posts,
+      ];
+    } else {
+      resultTypes.push({
+        results: aiResults.posts,
+        componentName: "search-result-topic",
+        type: "topic",
+        more: false,
+      });
+    }
+
+    this.search.results.resultTypes = resultTypes;
+    this.search.results = { ...this.search.results };
+    this.search.noResults = false;
+  }
+
+  <template>
+    <div
+      {{didUpdate this.onSearchTermChange this.search.activeGlobalSearchTerm}}
+      {{didUpdate this.checkAndAddAiResults this.totalResults}}
+      {{this.markAiResults}}
+    >
+      {{#if this.searchingWithAi}}
+        <div class="ai-quick-search-loading">
+          <div class="ai-quick-search-loading__content">
+            {{loadingSpinner}}
+          </div>
+        </div>
+      {{/if}}
+    </div>
+  </template>
+}
@@ -272,7 +272,6 @@ discourse_ai:
  ai_embeddings_semantic_quick_search_enabled:
    default: false
    client: true
-    hidden: true
    area: "ai-features/embeddings"
  ai_embeddings_semantic_search_hyde_persona:
    default: "-32"
@@ -126,73 +126,6 @@ module DiscourseAi
        candidates.where(id: candidate_topic_ids).pluck(:id)
      end

-      def quick_search(query)
-        max_semantic_results_per_page = 100
-        search = Search.new(query, { guardian: guardian })
-        search_term = search.term
-        hyde_model_id = self.class.find_ai_hyde_model_id
-
-        return [] if search_term.nil? || search_term.length < SiteSetting.min_search_term_length
-
-        vector = DiscourseAi::Embeddings::Vector.instance
-
-        digest = OpenSSL::Digest::SHA1.hexdigest(search_term)
-
-        embedding_key =
-          build_embedding_key(digest, hyde_model_id, SiteSetting.ai_embeddings_selected_model)
-
-        search_term_embedding =
-          Discourse
-            .cache
-            .fetch(embedding_key, expires_in: 1.week) do
-              vector.vector_from(search_term, asymmetric: true)
-            end
-
-        candidate_post_ids =
-          DiscourseAi::Embeddings::Schema
-            .for(Post)
-            .asymmetric_similarity_search(
-              search_term_embedding,
-              limit: max_semantic_results_per_page,
-              offset: 0,
-            )
-            .map(&:post_id)
-
-        semantic_results =
-          ::Post
-            .where(post_type: ::Topic.visible_post_types(guardian.user))
-            .public_posts
-            .where("topics.visible")
-            .where(id: candidate_post_ids)
-            .order("array_position(ARRAY#{candidate_post_ids}, posts.id)")
-
-        filtered_results = search.apply_filters(semantic_results)
-
-        rerank_posts_payload =
-          filtered_results
-            .map(&:cooked)
-            .map { Nokogiri::HTML5.fragment(_1).text }
-            .map { _1.truncate(2000, omission: "") }
-
-        reranked_results =
-          DiscourseAi::Inference::HuggingFaceTextEmbeddings.rerank(
-            search_term,
-            rerank_posts_payload,
-          )
-
-        reordered_ids = reranked_results.map { _1[:index] }.map { filtered_results[_1].id }.take(5)
-
-        reranked_semantic_results =
-          ::Post
-            .where(post_type: ::Topic.visible_post_types(guardian.user))
-            .public_posts
-            .where("topics.visible")
-            .where(id: reordered_ids)
-            .order("array_position(ARRAY#{reordered_ids}, posts.id)")
-
-        guardian.filter_allowed_categories(reranked_semantic_results)
-      end
-
      def hypothetical_post_from(search_term)
        context =
          DiscourseAi::Personas::BotContext.new(
@@ -216,4 +216,90 @@ describe DiscourseAi::Embeddings::EmbeddingsController do
      end
    end
  end
+
+  context "when performing a quick search" do
+    fab!(:vector_def, :open_ai_embedding_def)
+    fab!(:user)
+
+    before do
+      enable_current_plugin
+      SiteSetting.min_search_term_length = 3
+      SiteSetting.ai_embeddings_selected_model = vector_def.id
+      SiteSetting.ai_embeddings_semantic_quick_search_enabled = true
+      DiscourseAi::Embeddings::SemanticSearch.clear_cache_for("test")
+      SearchIndexer.enable
+      sign_in(user)
+    end
+
+    fab!(:topic)
+    fab!(:post) { Fabricate(:post, topic: topic, raw: "This is a test post") }
+
+    def index(topic)
+      vector = DiscourseAi::Embeddings::Vector.instance
+
+      stub_request(:post, "https://api.openai.com/v1/embeddings").to_return(
+        status: 200,
+        body: JSON.dump({ data: [{ embedding: [0.1] * 1536 }] }),
+      )
+
+      vector.generate_representation_from(topic)
+    end
+
+    def stub_embedding(query)
+      embedding = [0.049382] * 1536
+
+      EmbeddingsGenerationStubs.openai_service(
+        vector_def.lookup_custom_param("model_name"),
+        query,
+        embedding,
+      )
+    end
+
+    it "returns 400 when query is too short" do
+      get "/discourse-ai/embeddings/quick-search.json?q=ab"
+
+      expect(response.status).to eq(400)
+    end
+
+    it "returns results successfully with valid query" do
+      index(topic)
+      stub_embedding("test")
+
+      get "/discourse-ai/embeddings/quick-search.json?q=test"
+
+      expect(response.status).to eq(200)
+      expect(response.parsed_body["posts"]).to be_present
+      expect(response.parsed_body["topics"].map { |t| t["id"] }).to contain_exactly(topic.id)
+    end
+
+    context "when rate limiting is enabled" do
+      before { RateLimiter.enable }
+
+      it "rate limits non-cached queries" do
+        61.times do |i|
+          query = "test#{i}"
+          stub_embedding(query)
+          get "/discourse-ai/embeddings/quick-search.json?q=#{query}"
+        end
+
+        expect(response.status).to eq(429)
+      end
+
+      it "does not rate limit cached queries" do
+        query = "cached_query"
+        semantic_search = instance_double(DiscourseAi::Embeddings::SemanticSearch)
+        allow(DiscourseAi::Embeddings::SemanticSearch).to receive(:new).and_return(semantic_search)
+        allow(semantic_search).to receive(:cached_query?).with(query).and_return(true)
+        allow(semantic_search).to receive(:search_for_topics).with(
+          query,
+          1,
+          hyde: false,
+        ).and_return([])
+
+        100.times { get "/discourse-ai/embeddings/quick-search.json?q=#{query}" }
+
+        expect(response.status).to eq(200)
+      end
+    end
+  end
 end
@@ -0,0 +1,20 @@
+import { module, test } from "qunit";
+import AiQuickSearch from "discourse/plugins/discourse-ai/discourse/connectors/search-menu-results-bottom/ai-quick-search";
+
+module("Unit | Component | ai-quick-search", function () {
+  module("shouldRender", function () {
+    test("returns true when site setting is enabled", function (assert) {
+      const siteSettings = {
+        ai_embeddings_semantic_quick_search_enabled: true,
+      };
+      assert.true(AiQuickSearch.shouldRender({}, { siteSettings }));
+    });
+
+    test("returns false when site setting is disabled", function (assert) {
+      const siteSettings = {
+        ai_embeddings_semantic_quick_search_enabled: false,
+      };
+      assert.false(AiQuickSearch.shouldRender({}, { siteSettings }));
+    });
+  });
+});