FIX: LLM config must be valid before running a test (#34806)

<img width="500" height="246" alt="Screenshot 2025-09-15 at 10 34 05 AM"
src="https://github.com/user-attachments/assets/8d406f25-4720-4d63-9f0d-64031b64111b"
/>

plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -138,11 +138,15 @@ module DiscourseAi
       def test
         RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
 
-        llm_model = LlmModel.new(ai_llm_params)
+        # We don't care about the display_name attr for testing.
+        llm_model = LlmModel.new(ai_llm_params.merge(display_name: "LLM test"))
 
-        DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
-
-        render json: { success: true }
+        if llm_model.valid?
+          DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
+          render json: { success: true }
+        else
+          render json: { success: false, validation_errors: llm_model.errors.full_messages }
+        end
       rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
         render json: { success: false, error: e.message }
       end

plugins/discourse-ai/assets/javascripts/discourse/components/ai-llm-editor-form.gjs

@@ -26,6 +26,7 @@ export default class AiLlmEditorForm extends Component {
   @tracked testRunning = false;
   @tracked testResult = null;
   @tracked testError = null;
+  @tracked testValidationErrors = null;
 
   @cached
   get formData() {
@@ -100,7 +101,11 @@ export default class AiLlmEditorForm extends Component {
   }
 
   get testErrorMessage() {
-    return i18n("discourse_ai.llms.tests.failure", { error: this.testError });
+    if (this.testValidationErrors?.length > 0) {
+      return i18n("discourse_ai.llms.tests.invalid_config");
+    } else {
+      return i18n("discourse_ai.llms.tests.failure", { error: this.testError });
+    }
   }
 
   get displayTestResult() {
@@ -230,8 +235,10 @@ export default class AiLlmEditorForm extends Component {
 
       if (this.testResult) {
         this.testError = null;
+        this.testValidationErrors = null;
       } else {
         this.testError = configTestResult.error;
+        this.testValidationErrors = configTestResult.validation_errors;
       }
     } catch (e) {
       popupAjaxError(e);
@@ -622,6 +629,11 @@ export default class AiLlmEditorForm extends Component {
               <div class="ai-llm-editor-tests__failure">
                 {{icon "xmark"}}
                 {{this.testErrorMessage}}
+                <ul>
+                  {{#each this.testValidationErrors as |error|}}
+                    <li>{{error}}</li>
+                  {{/each}}
+                </ul>
               </div>
             {{/if}}
           </ConditionalLoadingSpinner>

plugins/discourse-ai/config/locales/client.en.yml

@@ -694,6 +694,7 @@ en:
           title: "Run test"
           running: "Running test..."
           success: "Success!"
+          invalid_config: "Fix the following errors in your configuration first:"
           failure: "Trying to contact the model returned this error: %{error}"
 
         hints:

plugins/discourse-ai/config/locales/server.en.yml

@@ -1,4 +1,10 @@
 en:
+  activerecord:
+    attributes:
+      llm_model:
+        name: "Model ID"
+        max_prompt_tokens: "Context window"
+
   discourse_automation:
     ai:
       flag_types:

plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb

@@ -486,6 +486,21 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
         end
       end
     end
+
+    context "when config is invalid" do
+      it "returns a success false with the validation error" do
+        get "/admin/plugins/discourse-ai/ai-llms/test.json",
+            params: {
+              ai_llm: test_attrs.except(:max_prompt_tokens),
+            }
+
+        expect(response).to be_successful
+        expect(response.parsed_body["success"]).to eq(false)
+        expect(response.parsed_body["validation_errors"]).to contain_exactly(
+          "Context window is not a number",
+        )
+      end
+    end
   end
 
   describe "DELETE #destroy" do