feat: add runtime payload support for evals

Author: omeraplak

.changeset/some-streets-sell.md

@@ -0,0 +1,6 @@
+---
+"@voltagent/evals": patch
+"@voltagent/core": patch
+---
+
+feat: add runtime payload support for evals

packages/core/src/eval/runtime/runtime.ts

@@ -1,5 +1,7 @@
 import { safeStringify } from "@voltagent/internal/utils";
 
+const RUNTIME_METADATA_KEY = "__runtime";
+
 export type SamplingPolicy =
   | { type: "always" }
   | { type: "never" }
@@ -199,6 +201,15 @@ export async function runLocalScorers<Payload extends Record<string, unknown>>(
 
     const durationMs = Date.now() - start;
 
+    const runtimeSnapshot: Record<string, unknown> = {
+      payload: cloneRecord(payload) ?? payload ?? null,
+      params: cloneRecord(scorerParams) ?? scorerParams ?? {},
+    };
+
+    metadata = mergeMetadata(metadata, {
+      [RUNTIME_METADATA_KEY]: runtimeSnapshot,
+    });
+
     const execution: LocalScorerExecutionResult = {
       id: definition.id,
       name: definition.name,

packages/evals/src/voltops/run.ts

@@ -424,10 +424,18 @@ function inferTerminalStatus(summary: ExperimentSummary): CompleteEvalRunRequest
   const hasErrors = summary.errorCount > 0;
   const hasFailures = summary.failureCount > 0;
   const criteriaEvaluations = summary.criteria ?? [];
-  const passedAllCriteria =
-    criteriaEvaluations.length === 0 || criteriaEvaluations.every((entry) => entry.passed);
+  const hasCriteria = criteriaEvaluations.length > 0;
+  const passedAllCriteria = criteriaEvaluations.every((entry) => entry.passed);
 
-  if (hasErrors || hasFailures || !passedAllCriteria) {
+  if (hasErrors) {
+    return "failed";
+  }
+
+  if (hasCriteria) {
+    return passedAllCriteria ? "succeeded" : "failed";
+  }
+
+  if (hasFailures) {
     return "failed";
   }
 

website/docs/evals/live-evaluations.md

@@ -176,15 +176,6 @@ const customScorer = buildScorer({
   .build();
 ```
 
-**Lazy-loaded scorer:**
-
-```ts
-scorer: async () => {
-  const { createAnswerCorrectnessScorer } = await import("@voltagent/scorers");
-  return createAnswerCorrectnessScorer();
-};
-```
-
 #### `params`
 
 Static or dynamic parameters passed to the scorer.