prepareCheckItems(DataItem dataItem) {
- return Collections.emptyList();
- }
-
- @Override
- protected String prepareUserPrompt(DataItem dataItem, int round) {
- return "";
- }
-
- @Override
- protected boolean needCheck(DataItem dataItem, int round) {
- return false;
- }
-
- @Override
- public boolean support(DataItem dataItem) {
- return false;
- }
-
- @Override
- public double getTotalScore() {
- return 0;
- }
- };
- }
-}
\ No newline at end of file
diff --git a/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/checker/model/CheckItemTest.java b/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/checker/model/CheckItemTest.java
deleted file mode 100644
index 62ebc59..0000000
--- a/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/checker/model/CheckItemTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-package com.evalkit.framework.eval.node.scorer.checker.model;
-
-import com.evalkit.framework.eval.node.scorer.checker.constants.CheckMethod;
-import org.junit.jupiter.api.Test;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.jupiter.api.Assertions.*;
-
-class CheckItemTest {
-
- // âââââââââââââââââââââââââââ é»è®€åŒéªè¯ ââââââââââââââââââââââââââââ
-
- @Test
- void defaultValues_areCorrect() {
- CheckItem item = CheckItem.builder().name("æ£æ¥é¡¹").build();
- assertEquals("æ£æ¥é¡¹", item.getName());
- assertEquals(1.0, item.getTotalScore(), 1e-6);
- assertEquals(1.0, item.getWeight(), 1e-6);
- assertFalse(item.isStar());
- assertTrue(item.isSupport());
- assertEquals(0.0, item.getDefaultScore(), 1e-6);
- assertFalse(item.isExecuted());
- assertEquals(CheckMethod.NONE, item.getCheckMethod());
- }
-
- // âââââââââââââââââââââââââââ åæ°æ ¡éª âââââââââââââââââââââââââââââ
-
- @Test
- void build_blankName_throwsIllegalArgument() {
- assertThatThrownBy(() -> CheckItem.builder().name("").build())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("äžèœäžºç©º");
- }
-
- @Test
- void build_negativeTotalScore_throwsIllegalArgument() {
- assertThatThrownBy(() -> CheckItem.builder().name("x").totalScore(-1).build())
- .isInstanceOf(IllegalArgumentException.class);
- }
-
- @Test
- void build_negativeWeight_throwsIllegalArgument() {
- assertThatThrownBy(() -> CheckItem.builder().name("x").weight(-0.1).build())
- .isInstanceOf(IllegalArgumentException.class);
- }
-
- @Test
- void build_negativeDefaultScore_throwsIllegalArgument() {
- assertThatThrownBy(() -> CheckItem.builder().name("x").defaultScore(-1).build())
- .isInstanceOf(IllegalArgumentException.class);
- }
-
- // âââââââââââââââââââââââââââ getWeightScore âââââââââââââââââââââââ
-
- @Test
- void getWeightScore_normalCase() {
- CheckItem item = CheckItem.builder().name("x").weight(2.0).build();
- item.setScore(0.8);
- assertEquals(1.6, item.getWeightScore(), 1e-6);
- }
-
- @Test
- void getWeightScore_zeroScore() {
- CheckItem item = CheckItem.builder().name("x").weight(3.0).build();
- item.setScore(0.0);
- assertEquals(0.0, item.getWeightScore(), 1e-6);
- }
-
- // âââââââââââââââââââââââââââ support=false æ¶åå§åæ°å defaultScore â
-
- @Test
- void support_false_scoreEqualsDefaultScore() {
- CheckItem item = CheckItem.builder()
- .name("x")
- .support(false)
- .defaultScore(0.5)
- .build();
- assertFalse(item.isSupport());
- assertEquals(0.5, item.getScore(), 1e-6);
- }
-
- // âââââââââââââââââââââââââââ star æ å¿ ââââââââââââââââââââââââââââ
-
- @Test
- void star_flag_isSetCorrectly() {
- CheckItem item = CheckItem.builder().name("å¿
è¿é¡¹").star(true).build();
- assertTrue(item.isStar());
- }
-
- // âââââââââââââââââââââââââââ setter/getter âââââââââââââââââââââââââ
-
- @Test
- void setters_workCorrectly() {
- CheckItem item = CheckItem.builder().name("item").build();
- item.setScore(0.9);
- item.setReason("æµè¯çç±");
- item.setExecuted(true);
- item.setCheckMethod(CheckMethod.LLM);
-
- assertEquals(0.9, item.getScore(), 1e-6);
- assertEquals("æµè¯çç±", item.getReason());
- assertTrue(item.isExecuted());
- assertEquals(CheckMethod.LLM, item.getCheckMethod());
- }
-
- // âââââââââââââââââââââââââââ checkDescription âââââââââââââââââââââ
-
- @Test
- void checkDescription_isSetAndRetrieved() {
- CheckItem item = CheckItem.builder()
- .name("x")
- .checkDescription("è¿æ¯æ£æ¥æè¿°")
- .build();
- assertEquals("è¿æ¯æ£æ¥æè¿°", item.getCheckDescription());
- }
-}
\ No newline at end of file
diff --git a/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/strategy/ScoreStrategyTest.java b/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/strategy/ScoreStrategyTest.java
deleted file mode 100644
index 5b9fb04..0000000
--- a/evalkit-eval/src/test/java/com/evalkit/framework/eval/node/scorer/strategy/ScoreStrategyTest.java
+++ /dev/null
@@ -1,247 +0,0 @@
-package com.evalkit.framework.eval.node.scorer.strategy;
-
-import com.evalkit.framework.eval.model.ScorerResult;
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/**
- * åè¯äŒ°åæ°çç¥åå
æµè¯
- *
- * èŠç: SumScoreStrategy / AvgScoreStrategy / MinScoreStrategy
- * AvgScoreRateStrategy / MaxScoreRateStrategy / MinScoreRateStrategy / SumScoreRateStrategy
- */
-class ScoreStrategyTest {
-
- // âââââââââââââââ èŸ
å©æ¹æ³ ââââââââââââââââââââââââââââââââââââââââ
-
- private ScorerResult r(double score, double scoreRate) {
- return ScorerResult.builder()
- .metric("m")
- .score(score)
- .scoreRate(scoreRate)
- .success(true)
- .build();
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // SumScoreStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void sumScore_normalCase() {
- SumScoreStrategy s = new SumScoreStrategy();
- List rs = Arrays.asList(r(0.8, 0.8), r(0.6, 0.6));
- assertEquals(1.4, s.calScore(rs), 1e-6);
- }
-
- @Test
- void sumScore_emptyList_returnsZero() {
- SumScoreStrategy s = new SumScoreStrategy();
- assertEquals(0.0, s.calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void sumScore_skipsFailedResults() {
- // SumScoreStrategy: ä»
对 success=true çç»ææ±å
- ScorerResult failed = ScorerResult.builder().metric("f").score(0.9).success(false).build();
- ScorerResult passed = ScorerResult.builder().metric("p").score(1.0).success(true).build();
- SumScoreStrategy s = new SumScoreStrategy();
- // failed äžè¢«è®¡å
¥ïŒisSuccess=false æ¶äžå ïŒ
- assertEquals(1.0, s.calScore(Arrays.asList(failed, passed)), 1e-6);
- }
-
- @Test
- void sumScore_strategyName() {
- assertEquals("åæ°æ±åçç¥", new SumScoreStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // AvgScoreStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void avgScore_normalCase() {
- AvgScoreStrategy s = new AvgScoreStrategy();
- List rs = Arrays.asList(r(0.8, 0.8), r(0.6, 0.6));
- assertEquals(0.7, s.calScore(rs), 1e-6);
- }
-
- @Test
- void avgScore_emptyList_returnsZero() {
- assertEquals(0.0, new AvgScoreStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void avgScore_singleElement() {
- assertEquals(0.9, new AvgScoreStrategy().calScore(Collections.singletonList(r(0.9, 0.9))), 1e-6);
- }
-
- @Test
- void avgScore_skipsNegativeScore() {
- // score=-1 çç»æ被跳è¿
- AvgScoreStrategy s = new AvgScoreStrategy();
- List rs = Arrays.asList(r(1.0, 1.0), r(-1.0, 0.0));
- // åªæ score=1.0 ææ â å¹³å = 1.0/1 = 1.0
- assertEquals(1.0, s.calScore(rs), 1e-6);
- }
-
- @Test
- void avgScore_strategyName() {
- assertEquals("å¹³ååæ°çç¥", new AvgScoreStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // MinScoreStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void minScore_normalCase() {
- MinScoreStrategy s = new MinScoreStrategy();
- List rs = Arrays.asList(r(0.8, 0.8), r(0.3, 0.3), r(1.0, 1.0));
- assertEquals(0.3, s.calScore(rs), 1e-6);
- }
-
- @Test
- void minScore_emptyList_returnsZero() {
- assertEquals(0.0, new MinScoreStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void minScore_singleElement() {
- assertEquals(0.7, new MinScoreStrategy().calScore(Collections.singletonList(r(0.7, 0.7))), 1e-6);
- }
-
- @Test
- void minScore_strategyName() {
- assertEquals("æå°åæ°çç¥", new MinScoreStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // AvgScoreRateStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void avgScoreRate_normalCase() {
- AvgScoreRateStrategy s = new AvgScoreRateStrategy();
- // (0.8 + 0.6) / 2 = 0.7
- List rs = Arrays.asList(r(0.8, 0.8), r(0.6, 0.6));
- assertEquals(0.7, s.calScore(rs), 1e-6);
- }
-
- @Test
- void avgScoreRate_emptyList_returnsZero() {
- assertEquals(0.0, new AvgScoreRateStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void avgScoreRate_singleElement() {
- assertEquals(0.5, new AvgScoreRateStrategy().calScore(Collections.singletonList(r(0.5, 0.5))), 1e-6);
- }
-
- @Test
- void avgScoreRate_strategyName() {
- assertEquals("å¹³ååŸåççç¥", new AvgScoreRateStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // MaxScoreRateStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void maxScoreRate_normalCase() {
- MaxScoreRateStrategy s = new MaxScoreRateStrategy();
- List rs = Arrays.asList(r(0.3, 0.3), r(0.9, 0.9), r(0.5, 0.5));
- assertEquals(0.9, s.calScore(rs), 1e-6);
- }
-
- @Test
- void maxScoreRate_emptyList_returnsZero() {
- assertEquals(0.0, new MaxScoreRateStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void maxScoreRate_strategyName() {
- assertEquals("æ倧åŸåççç¥", new MaxScoreRateStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // MinScoreRateStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void minScoreRate_emptyList_returnsZero() {
- assertEquals(0.0, new MinScoreRateStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void minScoreRate_strategyName() {
- assertEquals("æå°åŸåççç¥", new MinScoreRateStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // SumScoreRateStrategy
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void sumScoreRate_normalCase() {
- SumScoreRateStrategy s = new SumScoreRateStrategy();
- List rs = Arrays.asList(r(0.5, 0.5), r(0.7, 0.7));
- assertEquals(1.2, s.calScore(rs), 1e-6);
- }
-
- @Test
- void sumScoreRate_emptyList_returnsZero() {
- assertEquals(0.0, new SumScoreRateStrategy().calScore(Collections.emptyList()), 1e-6);
- }
-
- @Test
- void sumScoreRate_strategyName() {
- assertEquals("åŸåçæ±åçç¥", new SumScoreRateStrategy().getStrategyName());
- }
-
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- // ScoreStrategy ç±»åå€æ
- // âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
-
- @Test
- void sumScore_isScoreValueStrategy() {
- assertTrue(new SumScoreStrategy() instanceof ScoreValueStrategy);
- }
-
- @Test
- void avgScore_isScoreValueStrategy() {
- assertTrue(new AvgScoreStrategy() instanceof ScoreValueStrategy);
- }
-
- @Test
- void minScore_isScoreValueStrategy() {
- assertTrue(new MinScoreStrategy() instanceof ScoreValueStrategy);
- }
-
- @Test
- void avgScoreRate_isScoreRateStrategy() {
- assertTrue(new AvgScoreRateStrategy() instanceof ScoreRateStrategy);
- }
-
- @Test
- void maxScoreRate_isScoreRateStrategy() {
- assertTrue(new MaxScoreRateStrategy() instanceof ScoreRateStrategy);
- }
-
- @Test
- void minScoreRate_isScoreRateStrategy() {
- assertTrue(new MinScoreRateStrategy() instanceof ScoreRateStrategy);
- }
-
- @Test
- void sumScoreRate_isScoreRateStrategy() {
- assertTrue(new SumScoreRateStrategy() instanceof ScoreRateStrategy);
- }
-}
-
diff --git a/evalkit-eval/src/test/resources/dataItems.json b/evalkit-eval/src/test/resources/dataItems.json
deleted file mode 100644
index 082c1cb..0000000
--- a/evalkit-eval/src/test/resources/dataItems.json
+++ /dev/null
@@ -1,174 +0,0 @@
-[
- {
- "dataIndex": 0,
- "inputData": {
- "dataIndex": 0,
- "inputItem": {
- "query": "hello, å
宵è",
- "type": "1"
- }
- },
- "apiCompletionResult": {
- "dataIndex": 0,
- "resultItem": {
- "response": "Mock response for hello, å
宵è"
- },
- "startTime": 1763027533462,
- "endTime": 1763027533463,
- "timeCost": 1,
- "success": true
- },
- "evalResult": {
- "dataIndex": 0,
- "score": 0.0,
- "reason": "ç±äºçšæ·æ¥è¯¢ä¿¡æ¯äžå®æŽïŒçŒºå°å
·äœæ¥æãç®çå°çå
³é®èŠçŽ ïŒïŒAIå©æä»
æ瀺信æ¯çŒºå€±èæªäž»åšæšèæš¡ç³æ¡ä»¶äžçæºç¥šé项ïŒåŠå
šåœäœä»·ç¥šææ¥å£èªçïŒïŒä¹æªéè¿äº€äºåŒå¯Œçšæ·è¡¥å
ä¿¡æ¯ïŒå¯ŒèŽåšä¿¡æ¯å¡çå±ç€ºãçšæ·å奜å¹é
åææåå€çææè¯äŒ°ç»ŽåºŠååŸå䞺0ïŒåæ åºç³»ç»å¯¹æš¡ç³æ¥è¯¢çå€çèœåäžè¶³ã",
- "startTime": 0,
- "endTime": 0,
- "timeCost": 0,
- "scorerResults": [
- {
- "dataIndex": 0,
- "metric": "åŒåžžæµè¯",
- "score": 0.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "Error: / by zero",
- "extra": null,
- "statTime": 0,
- "endTime": 0,
- "timeCost": 0,
- "success": false,
- "pass": false,
- "threshold": 0.0,
- "star": false
- },
- {
- "dataIndex": 0,
- "metric": "åå€é¿åºŠæ£æ¥",
- "score": 1.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "hello, å
宵è çåå€é¿åºŠè¶
è¿5䞪å笊",
- "extra": null,
- "statTime": 1763027533557,
- "endTime": 1763027533558,
- "timeCost": 1,
- "success": true,
- "pass": true,
- "threshold": 0.0,
- "star": false
- },
- {
- "dataIndex": 0,
- "metric": "çžäŒŒåºŠæ£æ¥level1",
- "score": 0.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "çžäŒŒåºŠäžº0.0000ïŒå°äºéåŒ0.0000",
- "extra": {
- "similarity": 0.0,
- "similarityThreshold": 0.0
- },
- "statTime": 1763027533558,
- "endTime": 1763027533900,
- "timeCost": 342,
- "success": true,
- "pass": true,
- "threshold": 0.0,
- "star": false
- }
- ],
- "success": false,
- "pass": false,
- "threshold": 1.0,
- "scoreStrategyName": "æ倧åŸåççç¥"
- },
- "extra": null
- },
- {
- "dataIndex": 1,
- "inputData": {
- "dataIndex": 1,
- "inputItem": {
- "query": "hello, åœåºè",
- "type": "1"
- }
- },
- "apiCompletionResult": {
- "dataIndex": 1,
- "resultItem": {
- "response": "Mock response for hello, åœåºè"
- },
- "startTime": 1763027533463,
- "endTime": 1763027533463,
- "timeCost": 0,
- "success": true
- },
- "evalResult": {
- "dataIndex": 1,
- "score": 0.0,
- "reason": "åšç«èœŠç¥šæšèåºæ¯äžïŒç±äºçšæ·ä»
æäŸåºåå°åç®çå°èæªæ确任äœå奜ïŒåŠèœŠæ¬¡ç±»åã座äœç级ãæ¶éŽèŠæ±çïŒïŒAIå©æä»
é»è®€æšèæ 座/硬座çæ®éå蜊ïŒæ¢æªäž»åšåŒå¯Œçšæ·è¡¥å
ä¿¡æ¯ïŒä¹æªå±ç€ºç¬Šååžžè§å奜çæšèæ¹æ¡ïŒåŠé«éãå§éºçïŒïŒå¯ŒèŽåšèœŠæ¬¡æšèãå奜å¹é
åææåå€çææè¯äŒ°ç»ŽåºŠååŸå䞺0ïŒåæ åºç³»ç»å¯¹åºç¡æ¥è¯¢çé»è®€æšèçç¥ååšçŒºé·ã",
- "startTime": 0,
- "endTime": 0,
- "timeCost": 0,
- "scorerResults": [
- {
- "dataIndex": 1,
- "metric": "åŒåžžæµè¯",
- "score": 0.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "Error: / by zero",
- "extra": null,
- "statTime": 0,
- "endTime": 0,
- "timeCost": 0,
- "success": false,
- "pass": false,
- "threshold": 0.0,
- "star": false
- },
- {
- "dataIndex": 1,
- "metric": "åå€é¿åºŠæ£æ¥",
- "score": 1.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "hello, åœåºè çåå€é¿åºŠè¶
è¿5䞪å笊",
- "extra": null,
- "statTime": 1763027533980,
- "endTime": 1763027533980,
- "timeCost": 0,
- "success": true,
- "pass": true,
- "threshold": 0.0,
- "star": false
- },
- {
- "dataIndex": 1,
- "metric": "çžäŒŒåºŠæ£æ¥level1",
- "score": 0.0,
- "scoreRate": 0.0,
- "totalScore": 1.0,
- "reason": "çžäŒŒåºŠäžº0.0000ïŒå°äºéåŒ0.0000",
- "extra": {
- "similarity": 0.0,
- "similarityThreshold": 0.0
- },
- "statTime": 1763027533900,
- "endTime": 1763027533917,
- "timeCost": 17,
- "success": true,
- "pass": true,
- "threshold": 0.0,
- "star": false
- }
- ],
- "success": false,
- "pass": false,
- "threshold": 1.0,
- "scoreStrategyName": "æ倧åŸåççç¥"
- },
- "extra": null
- }
-]
\ No newline at end of file
diff --git a/evalkit-eval/src/test/resources/travel_demo/scenario2_config.json b/evalkit-eval/src/test/resources/travel_demo/scenario2_config.json
deleted file mode 100644
index 9642655..0000000
--- a/evalkit-eval/src/test/resources/travel_demo/scenario2_config.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
- "scenarioId": "itinerary_transport_hotel_flow",
- "sparqlTemplate": "PREFIX travel: \nPREFIX rdfs: \n\nSELECT ?depCityName ?destCityName ?transportType ?transportNo ?hotelName ?roomName ?attractionName\nWHERE {\n ?depCity rdfs:label ?depCityName .\n\n ?destCity rdfs:label ?destCityName .\n\n ?transport travel:departure ?depCity ;\n travel:destination ?destCity ;\n travel:transportType ?transportType ;\n travel:transportNo ?transportNo .\n\n ?hotel travel:locatedIn ?destCity ;\n travel:hotelName ?hotelName .\n\n ?room travel:roomType ?roomType ;\n travel:roomName ?roomName .\n\n ?attr travel:locatedIn ?destCity ;\n travel:attractionName ?attractionName .\n\n FILTER(?depCity != ?destCity)\n\n FILTER(?depCityName != \"äžæµ·\")\n}",
- "minSimilarity": 0.15,
- "maxSimilarity": 0.85,
- "goldenCase": {
- "kgDataUsed": {
- "depCityName": "äžæµ·",
- "destCityName": "æéœ",
- "transportType": "é«é",
- "transportNo": "G321",
- "hotelName": "çç«äž»é¢å®¢æ ",
- "roomName": "竹æ亲åå¥æ¿",
- "attractionName": "倧çç«ç¹è²åºå°"
- },
- "dialogue": [
- {
- "turn": 1,
- "query": "æç®åžŠå©åå»æéœç©å 倩ïŒæä»ä¹å¿
æå¡æ¯ç¹æšèåïŒ",
- "expectedVars": [
- "attractionName"
- ]
- },
- {
- "turn": 2,
- "query": "ä»äžæµ·åºåïŒæä»ä¹æšèç亀éæ¹åŒåïŒ",
- "expectedVars": [
- "transportNo"
- ]
- },
- {
- "turn": 3,
- "query": "å°äºé£èŸ¹æäžäœåªéæ¯èŸæ¹äŸ¿ïŒ",
- "expectedVars": [
- "hotelName"
- ]
- },
- {
- "turn": 4,
- "query": "家åºæ¿è¿æåã",
- "expectedVars": [
- "roomName"
- ]
- },
- {
- "turn": 5,
- "query": "åž®ææåæç奜ç蜊祚åè¿äžªäº²åæ¿äžèµ·äžåå§ã",
- "expectedVars": [
- "transportNo",
- "roomName"
- ]
- }
- ]
- }
-}
\ No newline at end of file
diff --git a/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LLMServiceFactoryTest.java b/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LLMServiceFactoryTest.java
index 181df1d..0626680 100644
--- a/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LLMServiceFactoryTest.java
+++ b/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LLMServiceFactoryTest.java
@@ -1,34 +1,74 @@
package com.evalkit.framework.infra.service.llm;
-import com.evalkit.framework.common.utils.runtime.RuntimeEnvUtils;
-import com.evalkit.framework.infra.service.llm.config.DeepseekLLMServiceConfig;
import com.evalkit.framework.infra.service.llm.config.LLMServiceConfig;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
@Slf4j
class LLMServiceFactoryTest {
- String deepSeekToken = RuntimeEnvUtils.getPropertyFromResource("secret.properties", "deepseek-token");
+ /**
+ * æé äžäžªåºå®åå€ç mock LLMServiceïŒäžäŸèµä»»äœå€éšæå¡
+ */
+ private LLMService mockLLMService(String fixedReply) {
+ return new LLMService() {
+ @Override
+ public String chat(String prompt) {
+ return fixedReply;
+ }
- @Test
- public void test() {
- // 泚åDeepSeek_Test倧暡åæå¡
- LLMServiceFactory.registerLLMService("DeepSeek_Test", new LLMServiceFactory.LLMServiceBuilder() {
@Override
- public LLMService build(LLMServiceConfig config) {
- return new DeepSeekLLMService((DeepseekLLMServiceConfig) config);
+ public String getModel() {
+ return "mock-model";
}
- });
+ };
+ }
+
+ @Test
+ void testRegisterAndCreateLLMService() {
+ // äœ¿çš mock builder 泚åæå¡ïŒäžäŸèµä»»äœå€éš token æ HTTP 请æ±
+ LLMServiceFactory.registerLLMService("Mock_Test",
+ (LLMServiceFactory.LLMServiceBuilder)
+ config -> mockLLMService("hello from mock"));
// å建æå¡å®äŸ
- DeepseekLLMServiceConfig config = DeepseekLLMServiceConfig.builder()
- .apiToken(deepSeekToken)
- .build();
- LLMService llmService = LLMServiceFactory.createLLMService("DeepSeek_Test", config);
-
- String query = "hello";
- String reply = llmService.chat(query);
- log.info("llm service config:{}, query:{}, reply:{}", config, query, reply);
+ LLMService llmService = LLMServiceFactory.createLLMService("Mock_Test",
+ LLMServiceConfig.builder().model("mock-model").build());
+
+ assertNotNull(llmService, "å建ç LLMService äžåºäžº null");
+
+ // éªè¯ mock è°çšå¯ä»¥æ£åžžè¿åïŒèäžäŒçæ£åèµ· HTTP 请æ±
+ String reply = llmService.chat("hello");
+ assertEquals("hello from mock", reply, "mock LLMService åºè¿åé¢æçåºå®åå€");
+ log.info("llmService model:{}, reply:{}", llmService.getModel(), reply);
+ }
+
+ @Test
+ void testCreateUnregisteredServiceThrowsException() {
+ // 访é®æªæ³šåçæå¡å称ïŒåºæåº IllegalArgumentException
+ assertThrows(IllegalArgumentException.class,
+ () -> LLMServiceFactory.createLLMService("NonExistentService", null),
+ "访é®æªæ³šåæå¡åºæåº IllegalArgumentException");
+ }
+
+ @Test
+ void testRegisterOverwriteExistingService() {
+ // å
泚åäžäžªè¿å "v1" çæå¡
+ LLMServiceFactory.registerLLMService("Override_Test",
+ (LLMServiceFactory.LLMServiceBuilder)
+ config -> mockLLMService("v1"));
+ LLMService v1 = LLMServiceFactory.createLLMService("Override_Test",
+ LLMServiceConfig.builder().model("mock").build());
+ assertEquals("v1", v1.chat("test"));
+
+ // èŠç泚å䞺è¿å "v2" çæå¡
+ LLMServiceFactory.registerLLMService("Override_Test",
+ (LLMServiceFactory.LLMServiceBuilder)
+ config -> mockLLMService("v2"));
+ LLMService v2 = LLMServiceFactory.createLLMService("Override_Test",
+ LLMServiceConfig.builder().model("mock").build());
+ assertEquals("v2", v2.chat("test"), "èŠç泚ååïŒæ°æå¡åºè¿åæ°çåå€");
}
}
\ No newline at end of file
diff --git a/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LoadBalanceLLMServiceTest.java b/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LoadBalanceLLMServiceTest.java
index 59482cd..474bba0 100644
--- a/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LoadBalanceLLMServiceTest.java
+++ b/evalkit-infra/src/test/java/com/evalkit/framework/infra/service/llm/LoadBalanceLLMServiceTest.java
@@ -1,9 +1,6 @@
package com.evalkit.framework.infra.service.llm;
import com.evalkit.framework.common.utils.list.ListUtils;
-import com.evalkit.framework.common.utils.runtime.RuntimeEnvUtils;
-import com.evalkit.framework.infra.service.llm.config.DeepseekLLMServiceConfig;
-import com.evalkit.framework.infra.service.llm.config.LLMServiceConfig;
import com.evalkit.framework.infra.service.llm.config.LoadBalanceLLMServiceConfig;
import com.evalkit.framework.infra.service.llm.strategy.RoundRobinLoadBalanceStrategy;
import lombok.extern.slf4j.Slf4j;
@@ -11,34 +8,44 @@
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.*;
@Slf4j
class LoadBalanceLLMServiceTest {
LoadBalanceLLMService loadBalanceLLMService;
+ /**
+ * æé äžäžªåºå®è¿åæå®å
容ç mock LLMServiceïŒäžåèµ·ä»»äœ HTTP 请æ±
+ */
+ private LLMService mockLLMService(String model, String fixedReply) {
+ return new LLMService() {
+ @Override
+ public String chat(String prompt) {
+ return fixedReply;
+ }
+
+ @Override
+ public String getModel() {
+ return model;
+ }
+ };
+ }
+
@BeforeEach
void setUp() {
- String deepSeekToken = RuntimeEnvUtils.getPropertyFromResource("secret.properties", "deepseek-token");
-
- // 泚å
- LLMServiceFactory.registerLLMService("DeepSeek_Test1", (LLMServiceFactory.LLMServiceBuilder) config -> new DeepSeekLLMService((DeepseekLLMServiceConfig) config));
- LLMServiceFactory.registerLLMService("DeepSeek_Test2", (LLMServiceFactory.LLMServiceBuilder) config -> new DeepSeekLLMService((DeepseekLLMServiceConfig) config));
-
- // å建
- DeepseekLLMServiceConfig config = DeepseekLLMServiceConfig.builder()
- .apiToken(deepSeekToken)
- .build();
- LLMService llmService10 = LLMServiceFactory.createLLMService("DeepSeek_Test1", config);
- LLMService llmService11 = LLMServiceFactory.createLLMService("DeepSeek_Test1", config);
- LLMService llmService12 = LLMServiceFactory.createLLMService("DeepSeek_Test1", config);
- LLMService llmService13 = LLMServiceFactory.createLLMService("DeepSeek_Test1", config);
- LLMService llmService14 = LLMServiceFactory.createLLMService("DeepSeek_Test1", config);
- LLMService llmService20 = LLMServiceFactory.createLLMService("DeepSeek_Test2", config);
- LLMService llmService21 = LLMServiceFactory.createLLMService("DeepSeek_Test2", config);
-
- // èŽèœœ
- List llmServices = ListUtils.of(llmService10, llmService11, llmService12, llmService13, llmService14, llmService20, llmService21);
+ // çš mock LLMService æ¿ä»£çå®ç DeepSeek æå¡ïŒäžäŸèµå€éš token æ HTTP
+ LLMService llmService1 = mockLLMService("mock-model-1", "reply from model-1");
+ LLMService llmService2 = mockLLMService("mock-model-1", "reply from model-1");
+ LLMService llmService3 = mockLLMService("mock-model-1", "reply from model-1");
+ LLMService llmService4 = mockLLMService("mock-model-2", "reply from model-2");
+ LLMService llmService5 = mockLLMService("mock-model-2", "reply from model-2");
+
+ List llmServices = ListUtils.of(
+ llmService1, llmService2, llmService3, llmService4, llmService5);
+
loadBalanceLLMService = new LoadBalanceLLMService(
LoadBalanceLLMServiceConfig.builder()
.llmServices(llmServices)
@@ -48,10 +55,35 @@ void setUp() {
}
@Test
- void test() {
+ void testGetModel() {
String model = loadBalanceLLMService.getModel();
+ assertNotNull(model, "getModel() äžåºè¿å null");
log.info("models: {}", model);
- loadBalanceLLMService.chat("hello,world");
- loadBalanceLLMService.chat("ä»æ¥å€©æ°");
+ }
+
+ @Test
+ void testChatRoundRobin() {
+ // éªè¯èœ®è¯¢çç¥ïŒå€æ¬¡è°çšåºååžåšäžåæå¡äž
+ AtomicInteger callCount = new AtomicInteger(0);
+ for (int i = 0; i < 5; i++) {
+ String reply = loadBalanceLLMService.chat("test query " + i);
+ assertNotNull(reply, "chat() è¿åäžåºäžº null");
+ callCount.incrementAndGet();
+ }
+ assertEquals(5, callCount.get(), "åºæåå®æ 5 次 chat è°çš");
+ log.info("å®æ {} 次 chat è°çšïŒèŽèœœåè¡¡æ£åžž", callCount.get());
+ }
+
+ @Test
+ void testEmptyLLMServicesThrowsException() {
+ // æ ¡éªç©º services åè¡šæ¶æé åºæåºåŒåžž
+ assertThrows(IllegalArgumentException.class, () ->
+ new LoadBalanceLLMService(
+ LoadBalanceLLMServiceConfig.builder()
+ .llmServices(ListUtils.of())
+ .loadBalanceStrategy(new RoundRobinLoadBalanceStrategy())
+ .build()
+ )
+ );
}
}
\ No newline at end of file
diff --git a/evalkit-test/pom.xml b/evalkit-test/pom.xml
index 9a271c2..eca635b 100644
--- a/evalkit-test/pom.xml
+++ b/evalkit-test/pom.xml
@@ -40,6 +40,11 @@
HEAD
+
+
+ true
+
+
io.github.zendodx
diff --git a/evalkit-test/src/test/java/com/evalkit/framework/test/DAGEvalPerformanceTest.java b/evalkit-test/src/test/java/com/evalkit/framework/test/DAGEvalPerformanceTest.java
index b0f1e89..cf90394 100644
--- a/evalkit-test/src/test/java/com/evalkit/framework/test/DAGEvalPerformanceTest.java
+++ b/evalkit-test/src/test/java/com/evalkit/framework/test/DAGEvalPerformanceTest.java
@@ -24,7 +24,7 @@ public class DAGEvalPerformanceTest {
private static final Logger logger = LoggerFactory.getLogger(DAGEvalPerformanceTest.class);
private static final String tempDir = System.getProperty("java.io.tmpdir");
private static final String fileName = "DAGEvalTest_" + UuidUtils.generateUuid() + ".xlsx";
- public static int caseCount = 10000 * 6;
+ public static int caseCount = 10000;
private static PerformanceMonitor performanceMonitor;
/**
diff --git a/evalkit-test/src/test/java/com/evalkit/framework/test/DeltaEvalPerformanceTest.java b/evalkit-test/src/test/java/com/evalkit/framework/test/DeltaEvalPerformanceTest.java
index 1e552e9..eac5a55 100644
--- a/evalkit-test/src/test/java/com/evalkit/framework/test/DeltaEvalPerformanceTest.java
+++ b/evalkit-test/src/test/java/com/evalkit/framework/test/DeltaEvalPerformanceTest.java
@@ -24,7 +24,7 @@ public class DeltaEvalPerformanceTest {
private static final Logger logger = LoggerFactory.getLogger(DeltaEvalPerformanceTest.class);
private static final String tempDir = System.getProperty("java.io.tmpdir");
private static final String fileName = "DeltaEvalPerformanceTest_" + UuidUtils.generateUuid() + ".xlsx";
- public static int caseCount = 10000 * 5;
+ public static int caseCount = 10000;
private static PerformanceMonitor performanceMonitor;
/**
diff --git a/pom.xml b/pom.xml
index c335578..d20c037 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,6 +64,7 @@
1.5
3.2.5
1.7.3
+ 0.8.12
EvalKit Framework
+[](https://mvnrepository.com/artifact/io.github.zendodx/evalkit-eval)
+[](https://www.apache.org/licenses/LICENSE-2.0)
+[](https://www.oracle.com/java/)
+[](https://codecov.io/gh/zendodx/evalkit-framework)
+[](https://github.com/zendodx/evalkit-framework/stargazers)
+[](https://github.com/zendodx/evalkit-framework/forks)
+
+
##### ð äžæææ¡£ | ð [English Documentation](README_en.md)
## æŠè¿°
diff --git a/README_en.md b/README_en.md
index 9dc5e59..e763aca 100644
--- a/README_en.md
+++ b/README_en.md
@@ -1,5 +1,12 @@
#