From b10773c154df8e8fe078145c39a4f92ba7cec1fc Mon Sep 17 00:00:00 2001 From: Andrei Bratu Date: Thu, 13 Feb 2025 19:27:56 +0000 Subject: [PATCH 1/2] Align run utility signature with python sdk --- src/eval_utils/types.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/eval_utils/types.ts b/src/eval_utils/types.ts index 3a0f2308..f954fe8f 100644 --- a/src/eval_utils/types.ts +++ b/src/eval_utils/types.ts @@ -90,8 +90,10 @@ export interface Evaluator extends Identifiers { returnType?: EvaluatorReturnTypeEnum; /**The threshold to check the Evaluator against. If the aggregate value of the Evaluator is below this threshold, the check will fail.*/ threshold?: number; - callable: Function; - argsType: EvaluatorArgumentsType; + /**The function to run on the logs to produce the judgment - only required for local Evaluators.*/ + callable?: Function; + /**The type of arguments the Evaluator expects - only required for local Evaluators.*/ + argsType?: EvaluatorArgumentsType; } export interface TargetFreeEvaluator extends Evaluator { From d4d9f82e125ad1e999dc08db83c23a090a03a82f Mon Sep 17 00:00:00 2001 From: Andrei Bratu Date: Thu, 13 Feb 2025 19:39:26 +0000 Subject: [PATCH 2/2] allow online evaluators in run utility --- src/eval_utils/run.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/eval_utils/run.ts b/src/eval_utils/run.ts index 608ed3f6..5cdf6be2 100644 --- a/src/eval_utils/run.ts +++ b/src/eval_utils/run.ts @@ -285,7 +285,9 @@ export async function runEval( // Upsert the local Evaluators; other Evaluators are just referenced by `path` or `id` let localEvaluators: [EvaluatorResponse, Function][] = []; if (evaluators) { - const evaluatorsWithCallable = evaluators.filter((e) => e.callable !== null); + const evaluatorsWithCallable = evaluators.filter( + (e) => e.callable !== undefined, + ); if (evaluatorsWithCallable.length > 0 && function_ == null) { throw new Error(