class Riffer::Evals::RunResult

Represents the complete result of an evaluation run across multiple scenarios.

Attributes

Per-scenario evaluation results.

Public Class Methods

Source

# File lib/riffer/evals/run_result.rb, line 11
def initialize(scenario_results:)
  @scenario_results = scenario_results
end

Public Instance Methods

evaluator_token_usage ()

Source

# File lib/riffer/evals/run_result.rb, line 52
def evaluator_token_usage
  scenario_results.filter_map(&:evaluator_token_usage).reduce(:+)
end

Returns the summed token usage across every scenario’s LLM-as-judge evaluators, or nil when none reported usage.

scores ()

Source

# File lib/riffer/evals/run_result.rb, line 19
def scores
  return {} if scenario_results.empty?

  totals = Hash.new(0.0)
  counts = Hash.new(0)

  scenario_results.each do |scenario|
    scenario.scores.each do |evaluator, score|
      totals[evaluator] += score
      counts[evaluator] += 1
    end
  end

  averages = {} #: Hash[singleton(Riffer::Evals::Evaluator), Float]
  totals.each_with_object(averages) do |(evaluator, total), hash|
    hash[evaluator] = total / counts[evaluator]
  end
end

Returns average scores keyed by evaluator class across all scenarios.

to_h ()

Source

# File lib/riffer/evals/run_result.rb, line 60
def to_h
  {
    scores: scores.transform_keys(&:name),
    scenario_results: scenario_results.map(&:to_h),
    token_usage: token_usage&.to_h,
    evaluator_token_usage: evaluator_token_usage&.to_h,
  }
end

Returns a hash representation of the run result.

token_usage ()

Source

# File lib/riffer/evals/run_result.rb, line 43
def token_usage
  scenario_results.filter_map(&:token_usage).reduce(:+)
end

Returns the summed token usage the agents under test spent across every scenario, or nil when none reported usage.