mirror of
https://github.com/discourse/discourse.git
synced 2026-08-02 09:29:37 -05:00
REFACTOR: centralize eval orchestration around feature-driven playground (#35718)
This refactor collapses all eval execution paths into a single Playground orchestrator that understands feature-specific behavior, keeps StructuredRecorder usage consistent, and lets Eval act purely as a data loader. By tightening the flow and upgrading the specs and documentation, we gain clearer logs, simpler CLI wiring, and a cleaner seam for upcoming persona-backed work while preserving the existing eval surface area.
This commit is contained in:
Roman Rizzi
@@ -122,3 +122,8 @@ openapi/*
|
||||
|
||||
# direnv.net
|
||||
.direnv
|
||||
|
||||
# discourse-ai evals
|
||||
/plugins/discourse-ai/evals/cases
|
||||
/plugins/discourse-ai/evals/log
|
||||
/plugins/discourse-ai/config/eval-llms.local.yml
|
||||
|
||||
@@ -1,18 +1,29 @@
|
||||
# frozen_string_literal: true
|
||||
require "optparse"
|
||||
require_relative "features"
|
||||
|
||||
class DiscourseAi::Evals::Cli
|
||||
class Options
|
||||
attr_accessor :eval_name, :model, :list, :list_models
|
||||
def initialize(eval_name: nil, model: nil, list: false, list_models: false)
|
||||
attr_accessor :eval_name, :model, :list, :list_models, :list_features, :feature_key
|
||||
|
||||
def initialize(
|
||||
eval_name: nil,
|
||||
model: nil,
|
||||
list: false,
|
||||
list_models: false,
|
||||
list_features: false,
|
||||
feature_key: nil
|
||||
)
|
||||
@eval_name = eval_name
|
||||
@model = model
|
||||
@list = list
|
||||
@list_models = list_models
|
||||
@list_features = list_features
|
||||
@feature_key = feature_key
|
||||
end
|
||||
end
|
||||
|
||||
def self.parse_options!
|
||||
def self.parse_options!(features_registry)
|
||||
options = Options.new
|
||||
|
||||
parser =
|
||||
@@ -24,6 +35,9 @@ class DiscourseAi::Evals::Cli
|
||||
end
|
||||
|
||||
opts.on("--list-models", "List models") { |model| options.list_models = true }
|
||||
opts.on("--list-features", "List features available for evals") do
|
||||
options.list_features = true
|
||||
end
|
||||
|
||||
opts.on(
|
||||
"-m",
|
||||
@@ -32,6 +46,12 @@ class DiscourseAi::Evals::Cli
|
||||
) { |model| options.model = model }
|
||||
|
||||
opts.on("-l", "--list", "List evals") { |model| options.list = true }
|
||||
|
||||
opts.on(
|
||||
"-f",
|
||||
"--feature KEY",
|
||||
"Feature key to evaluate (module_name:feature_name)",
|
||||
) { |key| options.feature_key = key }
|
||||
end
|
||||
|
||||
show_help = ARGV.empty?
|
||||
@@ -42,6 +62,13 @@ class DiscourseAi::Evals::Cli
|
||||
exit 0
|
||||
end
|
||||
|
||||
if options.feature_key && !features_registry.valid_feature_key?(options.feature_key)
|
||||
STDERR.puts(
|
||||
"Unknown feature '#{options.feature_key}'. Run with --list-features to view valid keys.",
|
||||
)
|
||||
exit 1
|
||||
end
|
||||
|
||||
options
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,329 +1,111 @@
|
||||
#frozen_string_literal: true
|
||||
# frozen_string_literal: true
|
||||
|
||||
class DiscourseAi::Evals::Eval
|
||||
attr_reader :type,
|
||||
:path,
|
||||
:name,
|
||||
:description,
|
||||
:id,
|
||||
:args,
|
||||
:vision,
|
||||
:expected_output,
|
||||
:expected_output_regex,
|
||||
:expected_tool_call,
|
||||
:judge
|
||||
module DiscourseAi
|
||||
module Evals
|
||||
# Lightweight data object that loads eval definitions from disk.
|
||||
#
|
||||
# Each YAML file under `evals/cases` is parsed into an instance that exposes
|
||||
# metadata (id, description, feature) and the normalized args needed by the
|
||||
# Playground to execute the evaluation. The class intentionally performs no
|
||||
# business logic; it only validates and prepares the data for consumers.
|
||||
class Eval
|
||||
attr_reader :path,
|
||||
:name,
|
||||
:description,
|
||||
:id,
|
||||
:args,
|
||||
:vision,
|
||||
:feature,
|
||||
:expected_output,
|
||||
:expected_output_regex,
|
||||
:expected_tool_call,
|
||||
:judge
|
||||
|
||||
class EvalError < StandardError
|
||||
attr_reader :context
|
||||
class EvalError < StandardError
|
||||
attr_reader :context
|
||||
|
||||
def initialize(message, context)
|
||||
super(message)
|
||||
@context = context
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(path:)
|
||||
@yaml = YAML.load_file(path).symbolize_keys
|
||||
@path = path
|
||||
@name = @yaml[:name]
|
||||
@id = @yaml[:id]
|
||||
@description = @yaml[:description]
|
||||
@vision = @yaml[:vision]
|
||||
@type = @yaml[:type]
|
||||
@expected_output = @yaml[:expected_output]
|
||||
@expected_output_regex = @yaml[:expected_output_regex]
|
||||
@expected_output_regex =
|
||||
Regexp.new(@expected_output_regex, Regexp::MULTILINE) if @expected_output_regex
|
||||
@expected_tool_call = @yaml[:expected_tool_call]
|
||||
@expected_tool_call.symbolize_keys! if @expected_tool_call
|
||||
@judge = @yaml[:judge]
|
||||
@judge.symbolize_keys! if @judge
|
||||
if @yaml[:args].is_a?(Array)
|
||||
@args = @yaml[:args].map(&:symbolize_keys)
|
||||
else
|
||||
@args = @yaml[:args].symbolize_keys
|
||||
@args.each do |key, value|
|
||||
if (key.to_s.include?("_path") || key.to_s == "path") && value.is_a?(String)
|
||||
@args[key] = File.expand_path(File.join(File.dirname(path), value))
|
||||
def initialize(message, context)
|
||||
super(message)
|
||||
@context = context
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def run(llm:)
|
||||
result =
|
||||
case type
|
||||
when "helper"
|
||||
helper(llm, **args)
|
||||
when "pdf_to_text"
|
||||
pdf_to_text(llm, **args)
|
||||
when "image_to_text"
|
||||
image_to_text(llm, **args)
|
||||
when "prompt"
|
||||
DiscourseAi::Evals::PromptEvaluator.new(llm).prompt_call(args)
|
||||
when "edit_artifact"
|
||||
edit_artifact(llm, **args)
|
||||
when "summarization"
|
||||
summarization(llm, **args)
|
||||
CASES_GLOB = File.join(__dir__, "../cases", "*/*.yml")
|
||||
|
||||
# @return [Array<DiscourseAi::Evals::Eval>] all cases sorted by path so
|
||||
# the CLI emits a deterministic order.
|
||||
def self.available_cases
|
||||
Dir.glob(CASES_GLOB).sort.map { |path| new(path: path) }
|
||||
end
|
||||
|
||||
classify_results(result)
|
||||
rescue EvalError => e
|
||||
{ result: :fail, message: e.message, context: e.context }
|
||||
end
|
||||
# @param path [String] absolute path to the YAML definition file.
|
||||
# @raise [ArgumentError] when a required key (like `feature`) is missing.
|
||||
def initialize(path:)
|
||||
yaml = YAML.load_file(path).symbolize_keys
|
||||
@path = path
|
||||
@name = yaml[:name]
|
||||
@id = yaml[:id]
|
||||
@description = yaml[:description]
|
||||
@vision = yaml[:vision]
|
||||
@feature = yaml[:feature]
|
||||
if @feature.blank?
|
||||
raise ArgumentError, "Eval '#{@id || @name || path}' must define a 'feature' key."
|
||||
end
|
||||
@expected_output = yaml[:expected_output]
|
||||
@expected_output_regex = yaml[:expected_output_regex]
|
||||
@expected_output_regex =
|
||||
Regexp.new(@expected_output_regex, Regexp::MULTILINE) if @expected_output_regex
|
||||
@expected_tool_call = yaml[:expected_tool_call]
|
||||
@expected_tool_call.symbolize_keys! if @expected_tool_call
|
||||
@judge = yaml[:judge]
|
||||
@judge.symbolize_keys! if @judge
|
||||
|
||||
def print
|
||||
puts "#{id}: #{description}"
|
||||
end
|
||||
|
||||
def to_json
|
||||
{
|
||||
type: @type,
|
||||
path: @path,
|
||||
name: @name,
|
||||
description: @description,
|
||||
id: @id,
|
||||
args: @args,
|
||||
vision: @vision,
|
||||
expected_output: @expected_output,
|
||||
expected_output_regex: @expected_output_regex,
|
||||
}.compact
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# @param result [String, Array<Hash>] the result of the eval, either
|
||||
# "llm response" or [{ result: "llm response", other_attrs: here }]
|
||||
# @return [Array<Hash>] an array of hashes with the result classified
|
||||
# as pass or fail, along with extra attributes
|
||||
def classify_results(result)
|
||||
if result.is_a?(Array)
|
||||
result.each { |r| r.merge!(classify_result_pass_fail(r)) }
|
||||
else
|
||||
[classify_result_pass_fail(result)]
|
||||
end
|
||||
end
|
||||
|
||||
def classify_result_pass_fail(result)
|
||||
if expected_output
|
||||
if result == expected_output
|
||||
{ result: :pass }
|
||||
else
|
||||
{ result: :fail, expected_output: expected_output, actual_output: result }
|
||||
end
|
||||
elsif expected_output_regex
|
||||
if result.to_s.match?(expected_output_regex)
|
||||
{ result: :pass }
|
||||
else
|
||||
{ result: :fail, expected_output: expected_output_regex, actual_output: result }
|
||||
end
|
||||
elsif expected_tool_call
|
||||
tool_call = result
|
||||
|
||||
if result.is_a?(Array)
|
||||
tool_call = result.find { |r| r.is_a?(DiscourseAi::Completions::ToolCall) }
|
||||
end
|
||||
if !tool_call.is_a?(DiscourseAi::Completions::ToolCall) ||
|
||||
(tool_call.name != expected_tool_call[:name]) ||
|
||||
(tool_call.parameters != expected_tool_call[:params])
|
||||
{ result: :fail, expected_output: expected_tool_call, actual_output: result }
|
||||
else
|
||||
{ result: :pass }
|
||||
end
|
||||
elsif judge
|
||||
judge_result(result)
|
||||
else
|
||||
{ result: :pass }
|
||||
end
|
||||
end
|
||||
|
||||
def judge_result(result)
|
||||
prompt = judge[:prompt].dup
|
||||
if result.is_a?(String)
|
||||
prompt.sub!("{{output}}", result)
|
||||
args.each { |key, value| prompt.sub!("{{#{key}}}", value.to_s) }
|
||||
else
|
||||
prompt.sub!("{{output}}", result[:result])
|
||||
result.each { |key, value| prompt.sub!("{{#{key}}}", value.to_s) }
|
||||
end
|
||||
|
||||
prompt += <<~SUFFIX
|
||||
|
||||
Reply with a rating from 1 to 10, where 10 is perfect and 1 is terrible.
|
||||
|
||||
example output:
|
||||
|
||||
[RATING]10[/RATING] perfect output
|
||||
|
||||
example output:
|
||||
|
||||
[RATING]5[/RATING]
|
||||
|
||||
the following failed to preserve... etc...
|
||||
SUFFIX
|
||||
|
||||
judge_llm = DiscourseAi::Evals::Llm.choose(judge[:llm]).first
|
||||
|
||||
DiscourseAi::Completions::Prompt.new(
|
||||
"You are an expert judge tasked at testing LLM outputs.",
|
||||
messages: [{ type: :user, content: prompt }],
|
||||
)
|
||||
|
||||
result =
|
||||
judge_llm.llm_model.to_llm.generate(prompt, user: Discourse.system_user, temperature: 0)
|
||||
|
||||
if rating = result.match(%r{\[RATING\](\d+)\[/RATING\]})
|
||||
rating = rating[1].to_i
|
||||
end
|
||||
|
||||
if rating.to_i >= judge[:pass_rating]
|
||||
{ result: :pass }
|
||||
else
|
||||
{
|
||||
result: :fail,
|
||||
message: "LLM Rating below threshold, it was #{rating}, expecting #{judge[:pass_rating]}",
|
||||
context: result,
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def helper(llm, input:, name:, locale: nil)
|
||||
helper = DiscourseAi::AiHelper::Assistant.new(helper_llm: llm.llm_model)
|
||||
user = Discourse.system_user
|
||||
if locale
|
||||
user = User.new
|
||||
class << user
|
||||
attr_accessor :effective_locale
|
||||
@args =
|
||||
if !yaml.key?(:args) || yaml[:args].nil?
|
||||
{}
|
||||
elsif yaml[:args].is_a?(Array)
|
||||
yaml[:args].map(&:symbolize_keys)
|
||||
else
|
||||
normalize_args(path, yaml[:args])
|
||||
end
|
||||
end
|
||||
|
||||
user.effective_locale = locale
|
||||
user.admin = true
|
||||
end
|
||||
result =
|
||||
helper.generate_and_send_prompt(name, input, current_user = user, force_default_locale: false)
|
||||
|
||||
result[:suggestions].first
|
||||
end
|
||||
|
||||
def image_to_text(llm, path:)
|
||||
upload =
|
||||
UploadCreator.new(File.open(path), File.basename(path)).create_for(Discourse.system_user.id)
|
||||
|
||||
text = +""
|
||||
DiscourseAi::Utils::ImageToText
|
||||
.new(upload: upload, llm_model: llm.llm_model, user: Discourse.system_user)
|
||||
.extract_text do |chunk, error|
|
||||
text << chunk if chunk
|
||||
text << "\n\n" if chunk
|
||||
end
|
||||
text
|
||||
ensure
|
||||
upload.destroy if upload
|
||||
end
|
||||
|
||||
def pdf_to_text(llm, path:)
|
||||
upload =
|
||||
UploadCreator.new(File.open(path), File.basename(path)).create_for(Discourse.system_user.id)
|
||||
|
||||
text = +""
|
||||
DiscourseAi::Utils::PdfToText
|
||||
.new(upload: upload, user: Discourse.system_user, llm_model: llm.llm_model)
|
||||
.extract_text do |chunk|
|
||||
text << chunk if chunk
|
||||
text << "\n\n" if chunk
|
||||
def print
|
||||
puts "#{id}: #{description} (feature: #{feature})"
|
||||
end
|
||||
|
||||
text
|
||||
ensure
|
||||
upload.destroy if upload
|
||||
end
|
||||
# @return [Hash] plain data used by Recorder and CLI output.
|
||||
def to_json
|
||||
{
|
||||
path: @path,
|
||||
name: @name,
|
||||
description: @description,
|
||||
id: @id,
|
||||
feature: @feature,
|
||||
args: @args,
|
||||
vision: @vision,
|
||||
expected_output: @expected_output,
|
||||
expected_output_regex: @expected_output_regex,
|
||||
}.compact
|
||||
end
|
||||
|
||||
def edit_artifact(llm, css_path:, js_path:, html_path:, instructions_path:)
|
||||
css = File.read(css_path)
|
||||
js = File.read(js_path)
|
||||
html = File.read(html_path)
|
||||
instructions = File.read(instructions_path)
|
||||
artifact =
|
||||
AiArtifact.create!(
|
||||
css: css,
|
||||
js: js,
|
||||
html: html,
|
||||
user_id: Discourse.system_user.id,
|
||||
post_id: 1,
|
||||
name: "eval artifact",
|
||||
)
|
||||
private
|
||||
|
||||
post = Post.new(topic_id: 1, id: 1)
|
||||
diff =
|
||||
DiscourseAi::AiBot::ArtifactUpdateStrategies::Diff.new(
|
||||
llm: llm.llm_model.to_llm,
|
||||
post: post,
|
||||
user: Discourse.system_user,
|
||||
artifact: artifact,
|
||||
artifact_version: nil,
|
||||
instructions: instructions,
|
||||
)
|
||||
diff.apply
|
||||
|
||||
if diff.failed_searches.present?
|
||||
puts "Eval Errors encountered"
|
||||
p diff.failed_searches
|
||||
raise EvalError.new("Failed to apply all changes", diff.failed_searches)
|
||||
end
|
||||
|
||||
version = artifact.versions.last
|
||||
raise EvalError.new("Invalid JS", version.js) if !valid_javascript?(version.js)
|
||||
|
||||
output = { css: version.css, js: version.js, html: version.html }
|
||||
|
||||
artifact.destroy
|
||||
output
|
||||
end
|
||||
|
||||
def valid_javascript?(str)
|
||||
require "open3"
|
||||
|
||||
# Create a temporary file with the JavaScript code
|
||||
Tempfile.create(%w[test .js]) do |f|
|
||||
f.write(str)
|
||||
f.flush
|
||||
|
||||
File.write("/tmp/test.js", str)
|
||||
|
||||
begin
|
||||
Discourse::Utils.execute_command(
|
||||
"node",
|
||||
"--check",
|
||||
f.path,
|
||||
failure_message: "Invalid JavaScript syntax",
|
||||
timeout: 30, # reasonable timeout in seconds
|
||||
)
|
||||
true
|
||||
rescue Discourse::Utils::CommandError
|
||||
false
|
||||
# Converts relative paths (e.g. asset fixtures) into absolute ones so the
|
||||
# runner can execute from any working directory.
|
||||
#
|
||||
# @param path [String] original yaml file path used as the base directory.
|
||||
# @param values [Hash] raw args coming from the YAML file.
|
||||
# @return [Hash] symbolized args with absolute paths when relevant.
|
||||
def normalize_args(path, values)
|
||||
args = values.symbolize_keys
|
||||
args.each do |key, value|
|
||||
if (key.to_s.include?("_path") || key.to_s == "path") && value.is_a?(String)
|
||||
args[key] = File.expand_path(File.join(File.dirname(path), value))
|
||||
end
|
||||
end
|
||||
args
|
||||
end
|
||||
end
|
||||
rescue StandardError
|
||||
false
|
||||
end
|
||||
|
||||
def summarization(llm, input:)
|
||||
topic =
|
||||
Topic.new(
|
||||
category: Category.last,
|
||||
title: "Eval topic for topic summarization",
|
||||
id: -99,
|
||||
user_id: Discourse.system_user.id,
|
||||
)
|
||||
Post.new(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
|
||||
|
||||
strategy =
|
||||
DiscourseAi::Summarization::FoldContent.new(
|
||||
llm.llm_proxy,
|
||||
DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
|
||||
)
|
||||
|
||||
summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
|
||||
summary.summarized_text
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Evals
|
||||
class Features
|
||||
def initialize(modules: DiscourseAi::Configuration::Module.all, output: $stdout)
|
||||
@modules = modules
|
||||
@output = output
|
||||
end
|
||||
|
||||
def print
|
||||
module_entries.each do |module_name, entries|
|
||||
output.puts module_name
|
||||
|
||||
if entries.empty?
|
||||
output.puts " - no registered features"
|
||||
next
|
||||
end
|
||||
|
||||
entries.each { |entry| output.puts " - #{entry[:key]}" }
|
||||
end
|
||||
end
|
||||
|
||||
def feature_map(evals)
|
||||
grouped_evals = Array(evals).group_by { |eval| eval.feature }
|
||||
grouped_evals.transform_values { |mapped_evals| mapped_evals.map(&:id).sort }
|
||||
end
|
||||
|
||||
def feature_keys
|
||||
entries.map { |entry| entry[:key] }
|
||||
end
|
||||
|
||||
def valid_feature_key?(key)
|
||||
feature_keys.include?(key)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :modules, :output
|
||||
|
||||
def module_entries
|
||||
@module_entries ||= modules.map { |mod| [mod.name, entries_for_module(mod)] }
|
||||
end
|
||||
|
||||
def entries
|
||||
@entries ||= module_entries.flat_map { |(_, m_entries)| m_entries }
|
||||
end
|
||||
|
||||
def entries_for_module(mod)
|
||||
feature_entries_by_module[mod] ||= Array(mod.features).map do |feature|
|
||||
{ key: feature_key(mod, feature), module_name: mod.name }
|
||||
end
|
||||
end
|
||||
|
||||
def feature_entries_by_module
|
||||
@feature_entries_by_module ||= {}
|
||||
end
|
||||
|
||||
def feature_key(mod, feature)
|
||||
"#{mod.name}:#{feature.name}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,387 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "recorder"
|
||||
require_relative "eval"
|
||||
|
||||
module DiscourseAi
|
||||
module Evals
|
||||
# Coordinates the execution of eval cases against one or more LLMs.
|
||||
#
|
||||
# The Playground drives the orchestration loop: it prepares the Structured
|
||||
# Recorder, dispatches work to helpers/utilities based on the eval feature,
|
||||
# and feeds the aggregated results back to the Recorder. It intentionally
|
||||
# keeps higher-level scripts (`evals/run`) simple while centralizing
|
||||
# instrumentation and error handling.
|
||||
class Playground
|
||||
def initialize(output: $stdout)
|
||||
@output = output
|
||||
end
|
||||
|
||||
# Iterate through the provided LLM adapters and execute the eval case for
|
||||
# each one, recording structured logs along the way.
|
||||
#
|
||||
# @param eval_case [DiscourseAi::Evals::Eval] the scenario to run.
|
||||
# @param llms [Array<#name,#vision?,#llm_model>] LLM wrappers selected by the CLI.
|
||||
def run(eval_case:, llms:)
|
||||
recorder = Recorder.with_cassette(eval_case, output: output)
|
||||
|
||||
llms.each do |llm|
|
||||
start_time = Time.now.utc
|
||||
if eval_case.vision && !llm.vision?
|
||||
recorder.record_llm_skip(llm.name, "LLM does not support vision")
|
||||
next
|
||||
end
|
||||
|
||||
results = execute_eval(eval_case, llm)
|
||||
recorder.record_llm_results(llm.name, results, start_time)
|
||||
rescue DiscourseAi::Evals::Eval::EvalError => e
|
||||
recorder.record_llm_results(
|
||||
llm.name,
|
||||
[{ result: :fail, message: e.message, context: e.context }],
|
||||
start_time,
|
||||
)
|
||||
rescue StandardError => e
|
||||
recorder.record_llm_results(llm.name, [{ result: :fail, message: e.message }], start_time)
|
||||
end
|
||||
ensure
|
||||
recorder&.finish
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :output
|
||||
|
||||
HELPER_MODES = {
|
||||
"ai_helper:proofread" => DiscourseAi::AiHelper::Assistant::PROOFREAD,
|
||||
"ai_helper:explain" => DiscourseAi::AiHelper::Assistant::EXPLAIN,
|
||||
"ai_helper:smart_dates" => DiscourseAi::AiHelper::Assistant::REPLACE_DATES,
|
||||
"ai_helper:title_suggestions" => DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
|
||||
"ai_helper:markdown_tables" => DiscourseAi::AiHelper::Assistant::MARKDOWN_TABLE,
|
||||
"ai_helper:custom_prompt" => DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT,
|
||||
"ai_helper:translator" => DiscourseAi::AiHelper::Assistant::TRANSLATE,
|
||||
"ai_helper:image_caption" => DiscourseAi::AiHelper::Assistant::IMAGE_CAPTION,
|
||||
}.freeze
|
||||
|
||||
def execute_eval(eval_case, llm)
|
||||
feature = eval_case.feature
|
||||
|
||||
raw =
|
||||
if (helper_mode = helper_mode_for(feature))
|
||||
helper_args = eval_case.args
|
||||
unless helper_args.is_a?(Hash)
|
||||
raise ArgumentError,
|
||||
"Eval '#{eval_case.id}' must define helper args as a hash to use #{feature}"
|
||||
end
|
||||
helper(llm, helper_mode: helper_mode, **helper_args)
|
||||
elsif feature == "custom:pdf_to_text"
|
||||
pdf_to_text(llm, **eval_case.args)
|
||||
elsif feature == "custom:image_to_text"
|
||||
image_to_text(llm, **eval_case.args)
|
||||
elsif feature == "custom:prompt"
|
||||
DiscourseAi::Evals::PromptEvaluator.new(llm).prompt_call(eval_case.args)
|
||||
elsif feature == "custom:edit_artifact"
|
||||
edit_artifact(llm, **eval_case.args)
|
||||
elsif feature&.start_with?("summarization:")
|
||||
summarization(llm, **eval_case.args)
|
||||
else
|
||||
raise ArgumentError, "Unsupported eval feature '#{feature}'"
|
||||
end
|
||||
|
||||
classify_results(eval_case, raw)
|
||||
end
|
||||
|
||||
# @param feature [String] fully qualified feature key (module:feature).
|
||||
# @return [String, nil] the Assistant mode constant to use for helper runs.
|
||||
def helper_mode_for(feature)
|
||||
HELPER_MODES[feature]
|
||||
end
|
||||
|
||||
def classify_results(eval_case, result)
|
||||
if result.is_a?(Array)
|
||||
result.each { |r| r.merge!(classify_result(eval_case, r)) }
|
||||
else
|
||||
[classify_result(eval_case, result)]
|
||||
end
|
||||
end
|
||||
|
||||
def classify_result(eval_case, result)
|
||||
if eval_case.expected_output
|
||||
if result == eval_case.expected_output
|
||||
{ result: :pass }
|
||||
else
|
||||
{ result: :fail, expected_output: eval_case.expected_output, actual_output: result }
|
||||
end
|
||||
elsif eval_case.expected_output_regex
|
||||
if result.to_s.match?(eval_case.expected_output_regex)
|
||||
{ result: :pass }
|
||||
else
|
||||
{
|
||||
result: :fail,
|
||||
expected_output: eval_case.expected_output_regex,
|
||||
actual_output: result,
|
||||
}
|
||||
end
|
||||
elsif eval_case.expected_tool_call
|
||||
classify_tool_call(eval_case.expected_tool_call, result)
|
||||
elsif eval_case.judge
|
||||
judge_result(eval_case, result)
|
||||
else
|
||||
{ result: :pass }
|
||||
end
|
||||
end
|
||||
|
||||
def classify_tool_call(expected_tool_call, result)
|
||||
tool_call = result
|
||||
tool_call = result.find { |r| r.is_a?(DiscourseAi::Completions::ToolCall) } if result.is_a?(
|
||||
Array,
|
||||
)
|
||||
|
||||
if !tool_call.is_a?(DiscourseAi::Completions::ToolCall) ||
|
||||
tool_call.name != expected_tool_call[:name] ||
|
||||
tool_call.parameters != expected_tool_call[:params]
|
||||
{ result: :fail, expected_output: expected_tool_call, actual_output: result }
|
||||
else
|
||||
{ result: :pass }
|
||||
end
|
||||
end
|
||||
|
||||
def judge_result(eval_case, result)
|
||||
prompt = eval_case.judge[:prompt].dup
|
||||
|
||||
if result.is_a?(String)
|
||||
prompt.sub!("{{output}}", result)
|
||||
eval_case.args.each { |key, value| prompt.sub!("{{#{key}}}", value.to_s) }
|
||||
else
|
||||
prompt.sub!("{{output}}", result[:result])
|
||||
result.each { |key, value| prompt.sub!("{{#{key}}}", value.to_s) }
|
||||
end
|
||||
|
||||
prompt += <<~SUFFIX
|
||||
|
||||
Reply with a rating from 1 to 10, where 10 is perfect and 1 is terrible.
|
||||
|
||||
example output:
|
||||
|
||||
[RATING]10[/RATING] perfect output
|
||||
|
||||
example output:
|
||||
|
||||
[RATING]5[/RATING]
|
||||
|
||||
the following failed to preserve... etc...
|
||||
SUFFIX
|
||||
|
||||
judge_llm = DiscourseAi::Evals::Llm.choose(eval_case.judge[:llm]).first
|
||||
|
||||
DiscourseAi::Completions::Prompt.new(
|
||||
"You are an expert judge tasked at testing LLM outputs.",
|
||||
messages: [{ type: :user, content: prompt }],
|
||||
)
|
||||
|
||||
judge_result =
|
||||
judge_llm.llm_model.to_llm.generate(prompt, user: Discourse.system_user, temperature: 0)
|
||||
|
||||
rating_match = judge_result.match(%r{\[RATING\](\d+)\[/RATING\]})
|
||||
rating = rating_match ? rating_match[1].to_i : 0
|
||||
|
||||
if rating >= eval_case.judge[:pass_rating]
|
||||
{ result: :pass }
|
||||
else
|
||||
{
|
||||
result: :fail,
|
||||
message:
|
||||
"LLM Rating below threshold, it was #{rating}, expecting #{eval_case.judge[:pass_rating]}",
|
||||
context: judge_result,
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Execute an AI Helper prompt for the provided mode, handling optional
|
||||
# locale switching and custom prompts used by certain helper features.
|
||||
#
|
||||
# @param llm [#llm_model] helper-capable LLM wrapper.
|
||||
# @param helper_mode [String] Assistant mode constant (see HELPER_MODES).
|
||||
# @param input [String] user input for the helper.
|
||||
# @param locale [String, nil] optional locale to impersonate during the run.
|
||||
# @param extra [Hash] optional keyword args such as :custom_prompt.
|
||||
# @return [String] helper suggestion selected for evaluation.
|
||||
def helper(llm, helper_mode:, input:, locale: nil, **extra)
|
||||
helper = DiscourseAi::AiHelper::Assistant.new(helper_llm: llm.llm_model)
|
||||
user = Discourse.system_user
|
||||
|
||||
if locale
|
||||
user = User.new
|
||||
class << user
|
||||
attr_accessor :effective_locale
|
||||
end
|
||||
|
||||
user.effective_locale = locale
|
||||
user.admin = true
|
||||
end
|
||||
|
||||
force_default_locale = extra.fetch(:force_default_locale, false)
|
||||
custom_prompt = extra[:custom_prompt]
|
||||
|
||||
result =
|
||||
helper.generate_and_send_prompt(
|
||||
helper_mode,
|
||||
input,
|
||||
user,
|
||||
force_default_locale: force_default_locale,
|
||||
custom_prompt: custom_prompt,
|
||||
)
|
||||
|
||||
result[:suggestions].first
|
||||
end
|
||||
|
||||
# Extract text from an image upload by delegating to the ImageToText helper.
|
||||
#
|
||||
# @param llm [#llm_model] LLM wrapper backing the OCR step.
|
||||
# @param path [String] path to the source image used for OCR.
|
||||
# @return [String] text extracted from the image.
|
||||
def image_to_text(llm, path:)
|
||||
upload =
|
||||
UploadCreator.new(File.open(path), File.basename(path)).create_for(
|
||||
Discourse.system_user.id,
|
||||
)
|
||||
|
||||
text = +""
|
||||
DiscourseAi::Utils::ImageToText
|
||||
.new(upload: upload, llm_model: llm.llm_model, user: Discourse.system_user)
|
||||
.extract_text do |chunk, _error|
|
||||
text << chunk if chunk
|
||||
text << "\n\n" if chunk
|
||||
end
|
||||
text
|
||||
ensure
|
||||
upload.destroy if upload
|
||||
end
|
||||
|
||||
# Extract text from a PDF, optionally falling back to LLM-guided OCR for pages.
|
||||
#
|
||||
# @param llm [#llm_model] LLM wrapper passed to PdfToText for OCR guidance.
|
||||
# @param path [String] path to the PDF fixture.
|
||||
# @return [String] text aggregated across the PDF pages.
|
||||
def pdf_to_text(llm, path:)
|
||||
upload =
|
||||
UploadCreator.new(File.open(path), File.basename(path)).create_for(
|
||||
Discourse.system_user.id,
|
||||
)
|
||||
|
||||
text = +""
|
||||
DiscourseAi::Utils::PdfToText
|
||||
.new(upload: upload, user: Discourse.system_user, llm_model: llm.llm_model)
|
||||
.extract_text do |chunk|
|
||||
text << chunk if chunk
|
||||
text << "\n\n" if chunk
|
||||
end
|
||||
|
||||
text
|
||||
ensure
|
||||
upload.destroy if upload
|
||||
end
|
||||
|
||||
# Run the edit artifact flow, returning the final artifact contents.
|
||||
#
|
||||
# @param llm [#llm_model] LLM wrapper used to produce diffs.
|
||||
# @param css_path [String] path to the CSS fixture.
|
||||
# @param js_path [String] path to the JS fixture.
|
||||
# @param html_path [String] path to the HTML fixture.
|
||||
# @param instructions_path [String] instructions fed to the LLM.
|
||||
# @return [Hash] latest artifact snapshot ({ css:, js:, html: }).
|
||||
def edit_artifact(llm, css_path:, js_path:, html_path:, instructions_path:)
|
||||
css = File.read(css_path)
|
||||
js = File.read(js_path)
|
||||
html = File.read(html_path)
|
||||
instructions = File.read(instructions_path)
|
||||
artifact =
|
||||
AiArtifact.create!(
|
||||
css: css,
|
||||
js: js,
|
||||
html: html,
|
||||
user_id: Discourse.system_user.id,
|
||||
post_id: 1,
|
||||
name: "eval artifact",
|
||||
)
|
||||
|
||||
post = Post.new(topic_id: 1, id: 1)
|
||||
diff =
|
||||
DiscourseAi::AiBot::ArtifactUpdateStrategies::Diff.new(
|
||||
llm: llm.llm_model.to_llm,
|
||||
post: post,
|
||||
user: Discourse.system_user,
|
||||
artifact: artifact,
|
||||
artifact_version: nil,
|
||||
instructions: instructions,
|
||||
)
|
||||
diff.apply
|
||||
|
||||
if diff.failed_searches.present?
|
||||
raise DiscourseAi::Evals::Eval::EvalError.new(
|
||||
"Failed to apply all changes",
|
||||
diff.failed_searches,
|
||||
)
|
||||
end
|
||||
|
||||
version = artifact.versions.last
|
||||
unless valid_javascript?(version.js)
|
||||
raise DiscourseAi::Evals::Eval::EvalError.new("Invalid JS", version.js)
|
||||
end
|
||||
|
||||
output = { css: version.css, js: version.js, html: version.html }
|
||||
|
||||
artifact.destroy
|
||||
output
|
||||
end
|
||||
|
||||
def valid_javascript?(str)
|
||||
require "open3"
|
||||
|
||||
Tempfile.create(%w[test .js]) do |f|
|
||||
f.write(str)
|
||||
f.flush
|
||||
|
||||
begin
|
||||
Discourse::Utils.execute_command(
|
||||
"node",
|
||||
"--check",
|
||||
f.path,
|
||||
failure_message: "Invalid JavaScript syntax",
|
||||
timeout: 30,
|
||||
)
|
||||
true
|
||||
rescue Discourse::Utils::CommandError
|
||||
false
|
||||
end
|
||||
end
|
||||
rescue StandardError
|
||||
false
|
||||
end
|
||||
|
||||
# Summarize the supplied input by going through the Topic summarization flow.
|
||||
#
|
||||
# @param llm [#llm_model, #llm_proxy] wrapper providing access to the model.
|
||||
# @param input [String] text used to bootstrap the summarization context.
|
||||
# @return [String] generated summary text.
|
||||
def summarization(llm, input:)
|
||||
topic =
|
||||
Topic.new(
|
||||
category: Category.last,
|
||||
title: "Eval topic for topic summarization",
|
||||
id: -99,
|
||||
user_id: Discourse.system_user.id,
|
||||
)
|
||||
Post.new(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
|
||||
|
||||
strategy =
|
||||
DiscourseAi::Summarization::FoldContent.new(
|
||||
llm.llm_proxy,
|
||||
DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
|
||||
)
|
||||
|
||||
summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
|
||||
summary.summarized_text
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,126 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "fileutils"
|
||||
require "logger"
|
||||
require_relative "structured_logger"
|
||||
|
||||
module DiscourseAi
|
||||
module Evals
|
||||
class Recorder
|
||||
def self.with_cassette(an_eval, output: $stdout)
|
||||
logs_dir = File.join(__dir__, "../log")
|
||||
FileUtils.mkdir_p(logs_dir)
|
||||
|
||||
now = Time.now.strftime("%Y%m%d-%H%M%S")
|
||||
structured_log_filename = "#{an_eval.id}-#{now}.json"
|
||||
log_filename = "#{an_eval.id}-#{now}.log"
|
||||
|
||||
log_path = File.expand_path(File.join(logs_dir, log_filename))
|
||||
structured_log_path = File.expand_path(File.join(logs_dir, structured_log_filename))
|
||||
|
||||
logger = Logger.new(File.open(log_path, "a"))
|
||||
structured_logger = StructuredLogger.new(structured_log_path)
|
||||
|
||||
new(an_eval, logger, log_path, structured_logger, output: output).tap do |recorder|
|
||||
recorder.running
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(an_eval, logger, log_path, structured_logger, output: $stdout)
|
||||
@an_eval = an_eval
|
||||
@logger = logger
|
||||
@log_path = log_path
|
||||
@structured_logger = structured_logger
|
||||
@output = output
|
||||
end
|
||||
|
||||
def running
|
||||
attach_thread_loggers
|
||||
logger.info("Starting evaluation '#{an_eval.id}'")
|
||||
structured_logger.start_root(name: "Evaluating #{an_eval.id}", args: an_eval.to_json)
|
||||
end
|
||||
|
||||
def record_llm_skip(llm_name, reason)
|
||||
if !structured_logger.root_started?
|
||||
raise ArgumentError, "You didn't instantiated this object with #with_cassette"
|
||||
end
|
||||
logger.info("Skipping LLM: #{llm_name} - Reason: #{reason}")
|
||||
end
|
||||
|
||||
def record_llm_results(llm_name, results, start_time)
|
||||
if !structured_logger.root_started?
|
||||
raise ArgumentError, "You didn't instantiated this object with #with_cassette"
|
||||
end
|
||||
|
||||
llm_step = structured_logger.add_child_step(name: "Evaluating with LLM: #{llm_name}")
|
||||
|
||||
logger.info("Evaluating with LLM: #{llm_name}")
|
||||
output.puts "#{llm_name}: "
|
||||
|
||||
results.each do |result|
|
||||
if result[:result] == :fail
|
||||
output.puts "Failed 🔴"
|
||||
output.puts "Error: #{result[:message]}" if result[:message]
|
||||
# this is deliberate, it creates a lot of noise, but sometimes for debugging it's useful
|
||||
# output.puts "Context: #{result[:context].to_s[0..2000]}" if result[:context]
|
||||
if result[:expected_output] && result[:actual_output]
|
||||
output.puts "---- Expected ----\n#{result[:expected_output]}"
|
||||
output.puts "---- Actual ----\n#{result[:actual_output]}"
|
||||
end
|
||||
logger.error("Evaluation failed with LLM: #{llm_name}")
|
||||
logger.error("Error: #{result[:message]}") if result[:message]
|
||||
logger.error("Expected: #{result[:expected_output]}") if result[:expected_output]
|
||||
logger.error("Actual: #{result[:actual_output]}") if result[:actual_output]
|
||||
logger.error("Context: #{result[:context]}") if result[:context]
|
||||
elsif result[:result] == :pass
|
||||
output.puts "Passed 🟢"
|
||||
logger.info("Evaluation passed with LLM: #{llm_name}")
|
||||
else
|
||||
STDERR.puts "Error: Unknown result #{an_eval.inspect}"
|
||||
logger.error("Unknown result: #{an_eval.inspect}")
|
||||
end
|
||||
|
||||
structured_logger.append_entry(
|
||||
step: llm_step,
|
||||
name: result[:result] == :pass ? :good : :bad,
|
||||
started_at: start_time,
|
||||
ended_at: Time.now.utc,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def finish
|
||||
structured_logger.finish_root(end_time: Time.now.utc)
|
||||
|
||||
detach_thread_loggers
|
||||
|
||||
structured_logger.save
|
||||
|
||||
output.puts
|
||||
output.puts "Log file: #{log_path}"
|
||||
output.puts "Structured log file (ui.perfetto.dev): #{structured_logger.path}"
|
||||
ensure
|
||||
logger&.close
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :an_eval, :logger, :structured_logger, :output, :log_path
|
||||
|
||||
def attach_thread_loggers
|
||||
@previous_thread_loggers = {
|
||||
audit_log: Thread.current[:llm_audit_log],
|
||||
structured_log: Thread.current[:llm_audit_structured_log],
|
||||
}
|
||||
|
||||
Thread.current[:llm_audit_log] = logger
|
||||
Thread.current[:llm_audit_structured_log] = structured_logger
|
||||
end
|
||||
|
||||
def detach_thread_loggers
|
||||
Thread.current[:llm_audit_log] = @previous_thread_loggers[:audit_log]
|
||||
Thread.current[:llm_audit_structured_log] = @previous_thread_loggers[:structured_log]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,194 +0,0 @@
|
||||
#frozen_string_literal: true
|
||||
|
||||
class DiscourseAi::Evals::Runner
|
||||
class StructuredLogger
|
||||
def initialize
|
||||
@log = []
|
||||
@current_step = @log
|
||||
end
|
||||
|
||||
def log(name, args: nil, start_time: nil, end_time: nil)
|
||||
start_time ||= Time.now.utc
|
||||
end_time ||= Time.now.utc
|
||||
args ||= {}
|
||||
object = { name: name, args: args, start_time: start_time, end_time: end_time }
|
||||
@current_step << object
|
||||
end
|
||||
|
||||
def step(name, args: nil)
|
||||
start_time = Time.now.utc
|
||||
start_step = @current_step
|
||||
|
||||
new_step = { type: :step, name: name, args: args || {}, log: [], start_time: start_time }
|
||||
|
||||
@current_step << new_step
|
||||
@current_step = new_step[:log]
|
||||
yield new_step
|
||||
@current_step = start_step
|
||||
new_step[:end_time] = Time.now.utc
|
||||
end
|
||||
|
||||
def to_trace_event_json
|
||||
trace_events = []
|
||||
process_id = 1
|
||||
thread_id = 1
|
||||
|
||||
to_trace_event(@log, process_id, thread_id, trace_events)
|
||||
|
||||
JSON.pretty_generate({ traceEvents: trace_events })
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def to_trace_event(log_items, pid, tid, trace_events, parent_start_time = nil)
|
||||
log_items.each do |item|
|
||||
if item.is_a?(Hash) && item[:type] == :step
|
||||
trace_events << {
|
||||
name: item[:name],
|
||||
cat: "default",
|
||||
ph: "B", # Begin event
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
args: item[:args],
|
||||
ts: timestamp_in_microseconds(item[:start_time]),
|
||||
}
|
||||
|
||||
to_trace_event(item[:log], pid, tid, trace_events, item[:start_time])
|
||||
|
||||
trace_events << {
|
||||
name: item[:name],
|
||||
cat: "default",
|
||||
ph: "E", # End event
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
ts: timestamp_in_microseconds(item[:end_time]),
|
||||
}
|
||||
else
|
||||
trace_events << {
|
||||
name: item[:name],
|
||||
cat: "default",
|
||||
ph: "B",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
args: item[:args],
|
||||
ts: timestamp_in_microseconds(item[:start_time] || parent_start_time || Time.now.utc),
|
||||
s: "p", # Scope: process
|
||||
}
|
||||
trace_events << {
|
||||
name: item[:name],
|
||||
cat: "default",
|
||||
ph: "E",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
ts: timestamp_in_microseconds(item[:end_time] || Time.now.utc),
|
||||
s: "p",
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def timestamp_in_microseconds(time)
|
||||
(time.to_f * 1_000_000).to_i
|
||||
end
|
||||
end
|
||||
|
||||
attr_reader :llms, :cases
|
||||
|
||||
def self.evals_paths
|
||||
@eval_paths ||= Dir.glob(File.join(File.join(__dir__, "../cases"), "*/*.yml"))
|
||||
end
|
||||
|
||||
def self.evals
|
||||
@evals ||= evals_paths.map { |path| DiscourseAi::Evals::Eval.new(path: path) }
|
||||
end
|
||||
|
||||
def self.print
|
||||
evals.each(&:print)
|
||||
end
|
||||
|
||||
def initialize(eval_name:, llms:)
|
||||
@llms = llms
|
||||
@eval = self.class.evals.find { |c| c.id == eval_name }
|
||||
|
||||
if !@eval
|
||||
puts "Error: Unknown evaluation '#{eval_name}'"
|
||||
exit 1
|
||||
end
|
||||
|
||||
if @llms.empty?
|
||||
puts "Error: Unknown model 'model'"
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
def run!
|
||||
puts "Running evaluation '#{@eval.id}'"
|
||||
|
||||
structured_log_filename = "#{@eval.id}-#{Time.now.strftime("%Y%m%d-%H%M%S")}.json"
|
||||
log_filename = "#{@eval.id}-#{Time.now.strftime("%Y%m%d-%H%M%S")}.log"
|
||||
logs_dir = File.join(__dir__, "../log")
|
||||
FileUtils.mkdir_p(logs_dir)
|
||||
|
||||
log_path = File.expand_path(File.join(logs_dir, log_filename))
|
||||
structured_log_path = File.expand_path(File.join(logs_dir, structured_log_filename))
|
||||
|
||||
logger = Logger.new(File.open(log_path, "a"))
|
||||
logger.info("Starting evaluation '#{@eval.id}'")
|
||||
|
||||
Thread.current[:llm_audit_log] = logger
|
||||
structured_logger = Thread.current[:llm_audit_structured_log] = StructuredLogger.new
|
||||
|
||||
structured_logger.step("Evaluating #{@eval.id}", args: @eval.to_json) do
|
||||
llms.each do |llm|
|
||||
if @eval.vision && !llm.vision?
|
||||
logger.info("Skipping LLM: #{llm.name} as it does not support vision")
|
||||
next
|
||||
end
|
||||
|
||||
structured_logger.step("Evaluating with LLM: #{llm.name}") do |step|
|
||||
logger.info("Evaluating with LLM: #{llm.name}")
|
||||
print "#{llm.name}: "
|
||||
results = @eval.run(llm: llm)
|
||||
|
||||
results.each do |result|
|
||||
step[:args] = result
|
||||
step[:cname] = result[:result] == :pass ? :good : :bad
|
||||
|
||||
if result[:result] == :fail
|
||||
puts "Failed 🔴"
|
||||
puts "Error: #{result[:message]}" if result[:message]
|
||||
# this is deliberate, it creates a lot of noise, but sometimes for debugging it's useful
|
||||
#puts "Context: #{result[:context].to_s[0..2000]}" if result[:context]
|
||||
if result[:expected_output] && result[:actual_output]
|
||||
puts "---- Expected ----\n#{result[:expected_output]}"
|
||||
puts "---- Actual ----\n#{result[:actual_output]}"
|
||||
end
|
||||
logger.error("Evaluation failed with LLM: #{llm.name}")
|
||||
logger.error("Error: #{result[:message]}") if result[:message]
|
||||
logger.error("Expected: #{result[:expected_output]}") if result[:expected_output]
|
||||
logger.error("Actual: #{result[:actual_output]}") if result[:actual_output]
|
||||
logger.error("Context: #{result[:context]}") if result[:context]
|
||||
elsif result[:result] == :pass
|
||||
puts "Passed 🟢"
|
||||
logger.info("Evaluation passed with LLM: #{llm.name}")
|
||||
else
|
||||
STDERR.puts "Error: Unknown result #{eval.inspect}"
|
||||
logger.error("Unknown result: #{eval.inspect}")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
#structured_logger.save(structured_log_path)
|
||||
|
||||
File.write("#{structured_log_path}", structured_logger.to_trace_event_json)
|
||||
|
||||
puts
|
||||
puts "Log file: #{log_path}"
|
||||
puts "Structured log file (ui.perfetto.dev): #{structured_log_path}"
|
||||
|
||||
# temp code
|
||||
# puts File.read(structured_log_path)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,137 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "json"
|
||||
|
||||
module DiscourseAi
|
||||
module Evals
|
||||
class StructuredLogger
|
||||
attr_reader :root, :path
|
||||
|
||||
def initialize(path)
|
||||
@root = nil
|
||||
@path = path
|
||||
end
|
||||
|
||||
def start_root(name:, args: {})
|
||||
raise ArgumentError, "root already started" if root
|
||||
|
||||
@root = build_step(name: name, args: args)
|
||||
end
|
||||
|
||||
def root_started?
|
||||
!root.nil?
|
||||
end
|
||||
|
||||
def add_child_step(name:, args: {})
|
||||
ensure_root!
|
||||
child = build_step(name: name, args: args)
|
||||
root[:children] << child
|
||||
child
|
||||
end
|
||||
|
||||
def append_entry(step:, name:, args: {}, started_at: nil, ended_at: nil)
|
||||
entry = {
|
||||
name: name,
|
||||
args: args || {},
|
||||
start_time: started_at || current_time,
|
||||
end_time: ended_at || started_at || current_time,
|
||||
}
|
||||
|
||||
step[:entries] << entry
|
||||
entry
|
||||
end
|
||||
|
||||
def finish_root(end_time: nil)
|
||||
ensure_root!
|
||||
root[:end_time] = end_time || current_time
|
||||
end
|
||||
|
||||
def to_trace_event_json
|
||||
ensure_root!
|
||||
|
||||
trace_events = []
|
||||
emit_step(root, trace_events)
|
||||
JSON.pretty_generate({ traceEvents: trace_events })
|
||||
end
|
||||
|
||||
def save
|
||||
File.write(path, to_trace_event_json)
|
||||
end
|
||||
|
||||
def as_json
|
||||
ensure_root!
|
||||
root
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_step(name:, args:)
|
||||
{
|
||||
name: name,
|
||||
args: args || {},
|
||||
start_time: current_time,
|
||||
end_time: nil,
|
||||
entries: [],
|
||||
children: [],
|
||||
}
|
||||
end
|
||||
|
||||
def current_time
|
||||
Time.now.utc
|
||||
end
|
||||
|
||||
def ensure_root!
|
||||
raise ArgumentError, "root is not started" unless root
|
||||
end
|
||||
|
||||
def emit_step(step, trace_events, pid = 1, tid = 1)
|
||||
trace_events << {
|
||||
name: step[:name],
|
||||
cat: "default",
|
||||
ph: "B",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
args: step[:args],
|
||||
ts: timestamp_in_microseconds(step[:start_time]),
|
||||
}
|
||||
|
||||
step[:entries].each do |entry|
|
||||
trace_events << {
|
||||
name: entry[:name],
|
||||
cat: "default",
|
||||
ph: "B",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
args: entry[:args],
|
||||
ts: timestamp_in_microseconds(entry[:start_time]),
|
||||
s: "p",
|
||||
}
|
||||
trace_events << {
|
||||
name: entry[:name],
|
||||
cat: "default",
|
||||
ph: "E",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
ts: timestamp_in_microseconds(entry[:end_time]),
|
||||
s: "p",
|
||||
}
|
||||
end
|
||||
|
||||
step[:children].each { |child| emit_step(child, trace_events, pid, tid) }
|
||||
|
||||
trace_events << {
|
||||
name: step[:name],
|
||||
cat: "default",
|
||||
ph: "E",
|
||||
pid: pid,
|
||||
tid: tid,
|
||||
ts: timestamp_in_microseconds(step[:end_time] || current_time),
|
||||
}
|
||||
end
|
||||
|
||||
def timestamp_in_microseconds(time)
|
||||
(time.to_f * 1_000_000).to_i
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -3,25 +3,61 @@
|
||||
|
||||
require_relative "lib/boot"
|
||||
require_relative "lib/llm"
|
||||
require_relative "lib/cli"
|
||||
require_relative "lib/runner"
|
||||
require_relative "lib/eval"
|
||||
require_relative "lib/prompts/prompt_evaluator"
|
||||
require_relative "lib/prompts/single_test_runner"
|
||||
require_relative "lib/features"
|
||||
require_relative "lib/recorder"
|
||||
require_relative "lib/playground"
|
||||
require_relative "lib/cli"
|
||||
|
||||
options = DiscourseAi::Evals::Cli.parse_options!
|
||||
features_registry =
|
||||
DiscourseAi::Evals::Features.new(modules: DiscourseAi::Configuration::Module.all)
|
||||
playground = DiscourseAi::Evals::Playground.new(output: $stdout)
|
||||
|
||||
if options.list
|
||||
DiscourseAi::Evals::Runner.print
|
||||
exit 0
|
||||
end
|
||||
options = DiscourseAi::Evals::Cli.parse_options!(features_registry)
|
||||
|
||||
if options.list_models
|
||||
DiscourseAi::Evals::Llm.print
|
||||
exit 0
|
||||
end
|
||||
|
||||
DiscourseAi::Evals::Runner.new(
|
||||
eval_name: options.eval_name,
|
||||
llms: DiscourseAi::Evals::Llm.choose(options.model),
|
||||
).run!
|
||||
if options.list_features
|
||||
features_registry.print
|
||||
exit 0
|
||||
end
|
||||
|
||||
available_evals = DiscourseAi::Evals::Eval.available_cases
|
||||
|
||||
if options.list
|
||||
available_evals.each(&:print)
|
||||
exit 0
|
||||
end
|
||||
|
||||
llms = DiscourseAi::Evals::Llm.choose(options.model)
|
||||
|
||||
if llms.empty?
|
||||
puts "Error: Unknown model '#{options.model}'"
|
||||
exit 1
|
||||
end
|
||||
|
||||
selected_evals = available_evals
|
||||
selected_evals =
|
||||
selected_evals.select do |eval_case|
|
||||
eval_case.feature == options.feature_key
|
||||
end if options.feature_key.present?
|
||||
selected_evals =
|
||||
selected_evals.select do |eval_case|
|
||||
eval_case.id == options.eval_name
|
||||
end if options.eval_name.present?
|
||||
|
||||
if selected_evals.empty?
|
||||
if options.feature_key
|
||||
puts "Error: No evaluations registered for feature '#{options.feature_key}'"
|
||||
else
|
||||
puts "Error: Unknown evaluation '#{options.eval_name}'"
|
||||
end
|
||||
exit 1
|
||||
end
|
||||
|
||||
selected_evals.each { |eval_case| playground.run(eval_case: eval_case, llms: llms) }
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "../../evals/lib/eval"
|
||||
|
||||
RSpec.describe DiscourseAi::Evals::Eval do
|
||||
around do |example|
|
||||
Dir.mktmpdir do |dir|
|
||||
@cases_dir = dir
|
||||
stub_const(described_class, :CASES_GLOB, File.join(dir, "*/*.yml")) { example.run }
|
||||
end
|
||||
end
|
||||
|
||||
describe ".available_cases" do
|
||||
it "loads eval instances sorted by file path" do
|
||||
write_case("set-one", "second", "id" => "second", "feature" => "mod:second")
|
||||
write_case("set-one", "first", "id" => "first", "feature" => "mod:first")
|
||||
|
||||
cases = described_class.available_cases
|
||||
|
||||
expect(cases.map(&:id)).to eq(%w[first second])
|
||||
expect(cases).to all(be_a(described_class))
|
||||
end
|
||||
end
|
||||
|
||||
describe "#initialize" do
|
||||
it "raises when the feature key is missing" do
|
||||
path = write_case("invalid", "missing-feature", "feature" => "")
|
||||
|
||||
expect { described_class.new(path: path) }.to raise_error(
|
||||
ArgumentError,
|
||||
/must define a 'feature' key/,
|
||||
)
|
||||
end
|
||||
|
||||
it "expands relative *_path args to absolute paths" do
|
||||
folder = File.join(@cases_dir, "path-case")
|
||||
FileUtils.mkdir_p(folder)
|
||||
File.write(File.join(folder, "input.txt"), "hello world")
|
||||
|
||||
path =
|
||||
write_case(
|
||||
"path-case",
|
||||
"example",
|
||||
"args" => {
|
||||
"input_path" => "input.txt",
|
||||
"other" => "value",
|
||||
},
|
||||
)
|
||||
|
||||
eval_case = described_class.new(path: path)
|
||||
|
||||
expect(eval_case.args[:input_path]).to eq(File.expand_path(File.join(folder, "input.txt")))
|
||||
expect(eval_case.args[:other]).to eq("value")
|
||||
end
|
||||
|
||||
it "symbolizes array args elements" do
|
||||
path =
|
||||
write_case(
|
||||
"array-case",
|
||||
"example",
|
||||
"args" => [{ "prompt" => "Hello" }, { "expected_output" => "Hi" }],
|
||||
)
|
||||
|
||||
eval_case = described_class.new(path: path)
|
||||
|
||||
expect(eval_case.args).to eq([{ prompt: "Hello" }, { expected_output: "Hi" }])
|
||||
end
|
||||
|
||||
it "compiles expected_output_regex with multiline mode" do
|
||||
path = write_case("regex-case", "example", "expected_output_regex" => "line\\nnext")
|
||||
|
||||
eval_case = described_class.new(path: path)
|
||||
|
||||
expect(eval_case.expected_output_regex).to be_a(Regexp)
|
||||
expect(eval_case.expected_output_regex.source).to eq("line\\nnext")
|
||||
expect(eval_case.expected_output_regex.options & Regexp::MULTILINE).not_to eq(0)
|
||||
end
|
||||
|
||||
it "defaults args to an empty hash when not provided" do
|
||||
path = write_case("no-args", "example", "args" => nil)
|
||||
|
||||
eval_case = described_class.new(path: path)
|
||||
|
||||
expect(eval_case.args).to eq({})
|
||||
end
|
||||
end
|
||||
|
||||
def write_case(folder, name, overrides = {})
|
||||
case_dir = File.join(@cases_dir, folder)
|
||||
FileUtils.mkdir_p(case_dir)
|
||||
|
||||
data = {
|
||||
"id" => "#{name}-id",
|
||||
"name" => "#{name} name",
|
||||
"description" => "example description",
|
||||
"feature" => "module:#{name}",
|
||||
}.merge(overrides)
|
||||
|
||||
data["args"] = { "prompt" => "Hello" } unless overrides.key?("args")
|
||||
|
||||
path = File.join(case_dir, "#{name}.yml")
|
||||
File.write(path, data.to_yaml)
|
||||
path
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,98 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "../../evals/lib/features"
|
||||
require_relative "../../lib/configuration/module"
|
||||
require_relative "../../lib/configuration/feature"
|
||||
|
||||
RSpec.describe DiscourseAi::Evals::Features do
|
||||
subject(:features) { described_class.new(modules: modules, output: output) }
|
||||
|
||||
let(:modules) do
|
||||
[
|
||||
DiscourseAi::Configuration::Module.new(
|
||||
1,
|
||||
"module-1",
|
||||
enabled_by_setting: "setting-1",
|
||||
features: [
|
||||
DiscourseAi::Configuration::Feature.new("feature-1", "persona-1", 1, "module-1"),
|
||||
],
|
||||
),
|
||||
DiscourseAi::Configuration::Module.new(
|
||||
2,
|
||||
"module-2",
|
||||
enabled_by_setting: "setting-2",
|
||||
features: [
|
||||
DiscourseAi::Configuration::Feature.new("feature-2", "persona-2", 2, "module-2"),
|
||||
],
|
||||
),
|
||||
]
|
||||
end
|
||||
|
||||
let(:output) { StringIO.new }
|
||||
|
||||
describe "#feature_keys" do
|
||||
it "matches the features exposed by the configuration modules" do
|
||||
expected_keys =
|
||||
modules.flat_map { |mod| mod.features.map { |feature| "#{mod.name}:#{feature.name}" } }
|
||||
|
||||
expect(features.feature_keys).to match_array(expected_keys)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#valid_feature_key?" do
|
||||
let(:known_module) { modules.first }
|
||||
let(:known_feature) { known_module.features.first }
|
||||
let(:known_key) { "#{known_module.name}:#{known_feature.name}" }
|
||||
|
||||
it "returns true when the key matches a registered feature" do
|
||||
expect(features.valid_feature_key?(known_key)).to be(true)
|
||||
end
|
||||
|
||||
it "returns false for unknown features" do
|
||||
expect(features.valid_feature_key?("search:unknown")).to be(false)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#feature_map" do
|
||||
let(:evals) do
|
||||
[
|
||||
Struct.new(:id, :feature).new("eval-3", "module-1:feature-1"),
|
||||
Struct.new(:id, :feature).new("eval-1", "module-1:feature-1"),
|
||||
Struct.new(:id, :feature).new("eval-2", "module-2:feature-2"),
|
||||
]
|
||||
end
|
||||
|
||||
it "groups evaluations by feature and sorts their ids" do
|
||||
expect(features.feature_map(evals)).to eq(
|
||||
"module-1:feature-1" => %w[eval-1 eval-3],
|
||||
"module-2:feature-2" => %w[eval-2],
|
||||
)
|
||||
end
|
||||
|
||||
it "returns an empty hash when no evaluations are registered" do
|
||||
empty_features = described_class.new(modules: modules, output: output)
|
||||
|
||||
expect(empty_features.feature_map([])).to eq({})
|
||||
end
|
||||
end
|
||||
|
||||
describe "#print" do
|
||||
it "prints the configured modules and their features" do
|
||||
features.print
|
||||
|
||||
expect(output.string).to include("module-2\n")
|
||||
expect(output.string).to include(" - module-2:feature-2\n")
|
||||
end
|
||||
|
||||
it "prints a placeholder for modules without features" do
|
||||
empty_module =
|
||||
DiscourseAi::Configuration::Module.new(999, "custom", features: [], enabled_by_setting: nil)
|
||||
modules_with_empty = modules + [empty_module]
|
||||
custom_features = described_class.new(modules: modules_with_empty, output: output)
|
||||
|
||||
custom_features.print
|
||||
|
||||
expect(output.string).to include("custom\n - no registered features\n")
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,108 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "../../evals/lib/playground"
|
||||
require_relative "../../evals/lib/eval"
|
||||
require_relative "../../evals/lib/llm"
|
||||
require_relative "../../evals/lib/recorder"
|
||||
|
||||
RSpec.describe DiscourseAi::Evals::Playground do
|
||||
subject(:playground) { described_class.new(output: output) }
|
||||
|
||||
let(:output) { StringIO.new }
|
||||
let(:recorder) do
|
||||
instance_double(
|
||||
DiscourseAi::Evals::Recorder,
|
||||
record_llm_skip: nil,
|
||||
record_llm_results: nil,
|
||||
finish: nil,
|
||||
)
|
||||
end
|
||||
let(:eval_case) do
|
||||
instance_double(
|
||||
DiscourseAi::Evals::Eval,
|
||||
id: "example-eval",
|
||||
vision: requires_vision,
|
||||
feature: "custom:prompt",
|
||||
args: {
|
||||
},
|
||||
expected_output: nil,
|
||||
expected_output_regex: nil,
|
||||
expected_tool_call: nil,
|
||||
judge: nil,
|
||||
)
|
||||
end
|
||||
let(:requires_vision) { false }
|
||||
let(:llm) do
|
||||
instance_double("DiscourseAi::Evals::Llm", name: "gpt-4", vision?: llm_supports_vision)
|
||||
end
|
||||
let(:llm_supports_vision) { true }
|
||||
|
||||
before do
|
||||
allow(DiscourseAi::Evals::Recorder).to receive(:with_cassette).and_return(recorder)
|
||||
freeze_time
|
||||
end
|
||||
|
||||
describe "#run" do
|
||||
it "records results for each llm" do
|
||||
allow(playground).to receive(:execute_eval).and_return([{ result: :pass }]) # rubocop:disable RSpec/SubjectStub
|
||||
|
||||
playground.run(eval_case: eval_case, llms: [llm])
|
||||
|
||||
expect(DiscourseAi::Evals::Recorder).to have_received(:with_cassette).with(
|
||||
eval_case,
|
||||
output: output,
|
||||
)
|
||||
expect(recorder).to have_received(:record_llm_results).with(
|
||||
"gpt-4",
|
||||
[{ result: :pass }],
|
||||
Time.now.utc,
|
||||
)
|
||||
expect(recorder).to have_received(:finish)
|
||||
end
|
||||
|
||||
context "when the eval requires vision but the llm does not support it" do
|
||||
let(:requires_vision) { true }
|
||||
let(:llm_supports_vision) { false }
|
||||
|
||||
it "skips the llm and records the reason" do
|
||||
playground.run(eval_case: eval_case, llms: [llm])
|
||||
|
||||
expect(recorder).to have_received(:record_llm_skip).with(
|
||||
"gpt-4",
|
||||
"LLM does not support vision",
|
||||
)
|
||||
expect(recorder).to have_received(:finish)
|
||||
end
|
||||
end
|
||||
|
||||
context "when eval execution raises an EvalError" do
|
||||
it "records the failure with the error context" do
|
||||
error = DiscourseAi::Evals::Eval::EvalError.new("boom", { foo: "bar" })
|
||||
allow(playground).to receive(:execute_eval).and_raise(error) # rubocop:disable RSpec/SubjectStub
|
||||
|
||||
playground.run(eval_case: eval_case, llms: [llm])
|
||||
|
||||
expect(recorder).to have_received(:record_llm_results).with(
|
||||
"gpt-4",
|
||||
[{ result: :fail, message: "boom", context: { foo: "bar" } }],
|
||||
Time.now.utc,
|
||||
)
|
||||
expect(recorder).to have_received(:finish)
|
||||
end
|
||||
end
|
||||
|
||||
context "when eval execution raises an unexpected error" do
|
||||
it "records the failure with the exception message" do
|
||||
allow(playground).to receive(:execute_eval).and_raise(StandardError.new("kaboom")) # rubocop:disable RSpec/SubjectStub
|
||||
|
||||
playground.run(eval_case: eval_case, llms: [llm])
|
||||
|
||||
expect(recorder).to have_received(:record_llm_results).with(
|
||||
"gpt-4",
|
||||
[{ result: :fail, message: "kaboom" }],
|
||||
Time.now.utc,
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,130 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "../../evals/lib/recorder"
|
||||
require_relative "../../evals/lib/structured_logger"
|
||||
require_relative "../../evals/lib/eval"
|
||||
|
||||
RSpec.describe DiscourseAi::Evals::Recorder do
|
||||
subject(:recorder) do
|
||||
described_class.new(eval_case, logger, "/tmp/example.json", structured_logger, output: output)
|
||||
end
|
||||
|
||||
let(:eval_case) do
|
||||
instance_double("DiscourseAi::Evals::Eval", id: "example-eval", to_json: { foo: "bar" })
|
||||
end
|
||||
let(:logger) { instance_double(Logger, info: nil, error: nil) }
|
||||
let(:structured_logger) do
|
||||
instance_double(
|
||||
DiscourseAi::Evals::StructuredLogger,
|
||||
start_root: nil,
|
||||
root_started?: root_started,
|
||||
add_child_step: child_step,
|
||||
append_entry: nil,
|
||||
finish_root: nil,
|
||||
to_trace_event_json: "{}",
|
||||
path: "/tmp/example.json",
|
||||
)
|
||||
end
|
||||
let(:root_started) { true }
|
||||
let(:child_step) { {} }
|
||||
let(:output) { StringIO.new }
|
||||
|
||||
before do
|
||||
allow(recorder).to receive(:attach_thread_loggers) # rubocop:disable RSpec/SubjectStub
|
||||
allow(recorder).to receive(:detach_thread_loggers) # rubocop:disable RSpec/SubjectStub
|
||||
end
|
||||
|
||||
describe "#running" do
|
||||
it "starts a root structured log step for the eval" do
|
||||
recorder.running
|
||||
|
||||
expect(structured_logger).to have_received(:start_root).with(
|
||||
name: "Evaluating example-eval",
|
||||
args: {
|
||||
foo: "bar",
|
||||
},
|
||||
)
|
||||
expect(logger).to have_received(:info).with("Starting evaluation 'example-eval'")
|
||||
end
|
||||
end
|
||||
|
||||
describe "#record_llm_skip" do
|
||||
context "when structured logging has not started" do
|
||||
let(:root_started) { false }
|
||||
|
||||
it "raises an informative error" do
|
||||
expect { recorder.record_llm_skip("gpt-4", "vision-only feature") }.to raise_error(
|
||||
ArgumentError,
|
||||
"You didn't instantiated this object with #with_cassette",
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
it "logs the skip reason when the structured log is active" do
|
||||
recorder.record_llm_skip("gpt-4", "vision-only feature")
|
||||
|
||||
expect(logger).to have_received(:info).with(
|
||||
"Skipping LLM: gpt-4 - Reason: vision-only feature",
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#record_llm_results" do
|
||||
let(:results) do
|
||||
[
|
||||
{ result: :pass },
|
||||
{
|
||||
result: :fail,
|
||||
message: "Mismatch",
|
||||
expected_output: "ideal",
|
||||
actual_output: "oops",
|
||||
context: "details",
|
||||
},
|
||||
]
|
||||
end
|
||||
let(:start_time) { Time.utc(2024, 1, 1, 12, 0, 0) }
|
||||
let(:now) { Time.utc(2024, 1, 1, 12, 1, 0) }
|
||||
|
||||
before { allow(Time).to receive(:now).and_return(now) }
|
||||
|
||||
context "when structured logging has not started" do
|
||||
let(:root_started) { false }
|
||||
|
||||
it "raises an informative error" do
|
||||
expect { recorder.record_llm_results("gpt-4", results, start_time) }.to raise_error(
|
||||
ArgumentError,
|
||||
"You didn't instantiated this object with #with_cassette",
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
it "records structured log entries and prints human friendly output" do
|
||||
recorder.record_llm_results("gpt-4", results, start_time)
|
||||
|
||||
expect(structured_logger).to have_received(:add_child_step).with(
|
||||
name: "Evaluating with LLM: gpt-4",
|
||||
)
|
||||
expect(structured_logger).to have_received(:append_entry).with(
|
||||
step: child_step,
|
||||
name: :good,
|
||||
started_at: start_time,
|
||||
ended_at: now.utc,
|
||||
)
|
||||
expect(structured_logger).to have_received(:append_entry).with(
|
||||
step: child_step,
|
||||
name: :bad,
|
||||
started_at: start_time,
|
||||
ended_at: now.utc,
|
||||
)
|
||||
|
||||
expect(logger).to have_received(:info).with("Evaluating with LLM: gpt-4")
|
||||
expect(logger).to have_received(:error).with("Evaluation failed with LLM: gpt-4")
|
||||
|
||||
expect(output.string).to include("gpt-4: ")
|
||||
expect(output.string).to include("Passed 🟢")
|
||||
expect(output.string).to include("Failed 🔴")
|
||||
expect(output.string).to include("---- Expected ----\nideal")
|
||||
expect(output.string).to include("---- Actual ----\noops")
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user