Skip to content

[improve/#336] MMR 추천 하이퍼파라미터 최적화 및 평가 체계 개선 #337

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Dimo-2562 merged 9 commits into from
Mar 8, 2026
Merged

[improve/#336] MMR 추천 하이퍼파라미터 최적화 및 평가 체계 개선 #337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bc2b675
improve: 추천 가중치 시나리오 결과를 json으로 저장
Dimo-2562 Mar 8, 2026
4591217
improve: 추천 평가에서도 애플리케이션과 동일하게 post 인덱스 세그먼트를 합치고, warmup하도록 변경
Dimo-2562 Mar 8, 2026
d61643c
test: knn 사이즈 비교 결과 리포트
Dimo-2562 Mar 8, 2026
42cf8da
test: knn 후보군의 크기를 3배로 하여 재 테스트
Dimo-2562 Mar 8, 2026
edb8610
chore: 테스트 logback 설정
Dimo-2562 Mar 8, 2026
9065d03
improve: 랜덤 샘플링을 인자로 받게 하여 on-off가 가능하도록 변경
Dimo-2562 Mar 8, 2026
c8744bc
test: MmmrCandidateSize 테스트 진행
Dimo-2562 Mar 8, 2026
e5e949e
test: Lambda값 테스트 진행
Dimo-2562 Mar 8, 2026
c71ec33
chore: 추천 하이퍼 파라미터 값을 테스트 결과에 맞게 변동
Dimo-2562 Mar 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/main/java/com/techfork/domain/recommendation/config/RecommendationProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,20 @@
@ConfigurationProperties(prefix = "recommendation")
public class RecommendationProperties {

private Integer knnSearchSize = 80;
private Integer knnSearchSize = 50;

private Integer numCandidates = 180;
private Integer numCandidates = 150;

private Integer mmrCandidateSize = 80;
private Integer mmrCandidateSize = 60;

private Integer mmrFinalSize = 30;

private Double lambda = 0.95;

private Integer mmrFirstTopK = 5;

private Integer mmrTopK = 3;

private Integer activeUserHours = 24;

private EmbeddingWeights embeddingWeights = new EmbeddingWeights();
Expand All @@ -36,8 +40,8 @@ public class RecommendationProperties {
@NoArgsConstructor
@AllArgsConstructor
public static class EmbeddingWeights {
private Float title = 0.4f;
private Float summary = 0.4f;
private Float title = 0.6f;
private Float summary = 0.2f;
private Float content = 0.2f;
}

Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/techfork/domain/recommendation/service/MmrService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ public List<MmrResult> applyMmr(List<MmrCandidate> candidates) {
log.debug("MMR 선택 시작: candidates={}, finalSize={}, lambda={}",
candidates.size(), finalSize, lambda);

// 첫 번째는 상위 K개 중에서 랜덤하게 선택 (다양성 증가)
int topK = Math.min(5, remainingCandidates.size());
int randomIndex = random.nextInt(topK);
// 첫 번째는 상위 K개 중에서 랜덤하게 선택 (다양성 증가, mmrFirstTopK=1이면 결정적)
int topK = Math.min(properties.getMmrFirstTopK(), remainingCandidates.size());
int randomIndex = topK <= 1 ? 0 : random.nextInt(topK);
MmrCandidate first = remainingCandidates.remove(randomIndex);
selectedResults.add(MmrResult.builder()
.postId(first.getPostId())
Expand All @@ -99,9 +99,9 @@ public List<MmrResult> applyMmr(List<MmrCandidate> candidates) {
// MMR 점수 내림차순 정렬
scoredCandidates.sort((a, b) -> Double.compare(b.mmrScore, a.mmrScore));

// 상위 K개 중에서 랜덤 선택
int topKForSelection = Math.min(3, scoredCandidates.size());
int randomIdx = random.nextInt(topKForSelection);
// 상위 K개 중에서 랜덤 선택 (mmrTopK=1이면 결정적)
int topKForSelection = Math.min(properties.getMmrTopK(), scoredCandidates.size());
int randomIdx = topKForSelection <= 1 ? 0 : random.nextInt(topKForSelection);
ScoredCandidate selected = scoredCandidates.get(randomIdx);

remainingCandidates.remove(selected.originalIndex);
Expand Down
73 changes: 49 additions & 24 deletions src/test/java/com/techfork/evaluation/recommendation/KValueComparisonTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
@Slf4j
public class KValueComparisonTest extends RecommendationTestBase {

private static final String REPORT_FILE = "evaluation-report-recommendation-phase2.json";

@Test
@DisplayName("knnSearchSize와 numCandidates 값 비교 평가")
void compareKValues() {
log.info("===== K 값에 따른 성능 및 품질 비교 =====");
@DisplayName("knnSearchSize와 numCandidates 값 비교 평가 (1차 후보군, MMR bypass)")
void compareKValues() throws Exception {
log.info("===== K 값에 따른 성능 및 품질 비교 (1차 후보군, MMR bypass) =====");
log.info("Ground-Truth: {} 명 사용자", cachedGroundTruth.size());

List<KConfig> kConfigs = createKConfigs();
Expand All @@ -31,30 +33,30 @@ void compareKValues() {
printKComparisonHeader();
List<KResult> results = evaluateAllKConfigs(kConfigs, testUsers);
printBestKResult(results);

// JSON 리포트 저장
saveKValueReport(REPORT_FILE, "K값 성능 비교 (1차 후보군)", false, toReportEntries(results));
}

/**
* 테스트할 K 값 설정 생성
*/
private List<KConfig> createKConfigs() {
return Arrays.asList(
// 현재 기본값
KConfig.builder().name("현재 (50/100)")
.knnSearchSize(50).numCandidates(100).build(),
KConfig.builder().name("소형 (30/90)")
.knnSearchSize(30).numCandidates(90).build(),

KConfig.builder().name("중간-하 (60/120)")
.knnSearchSize(60).numCandidates(120).build(),
KConfig.builder().name("중간-하 (40/120)")
.knnSearchSize(40).numCandidates(120).build(),

// 중간 값
KConfig.builder().name("중간 (70/150)")
.knnSearchSize(70).numCandidates(150).build(),
KConfig.builder().name("현재 (50/150)")
.knnSearchSize(50).numCandidates(150).build(),

KConfig.builder().name("중간-상 (80/180)")
.knnSearchSize(80).numCandidates(180).build(),
KConfig.builder().name("중간 (60/180)")
.knnSearchSize(60).numCandidates(180).build(),

// 이전 값
KConfig.builder().name("이전 (100/200)")
.knnSearchSize(100).numCandidates(200).build()
KConfig.builder().name("중간-상 (70/210)")
.knnSearchSize(70).numCandidates(210).build()
);
}

Expand All @@ -75,16 +77,16 @@ private List<KResult> evaluateAllKConfigs(List<KConfig> kConfigs, List<User> tes
properties.setMmrFinalSize(30);
properties.setLambda(1.0); // 다양성 제외, 관련성만

// 가중치는 최적값으로 고정 (제목+요약 중심)
// 가중치는 최적값으로 고정 (제목 중심)
RecommendationProperties.EmbeddingWeights weights = new RecommendationProperties.EmbeddingWeights();
weights.setTitle(0.4f);
weights.setSummary(0.4f);
weights.setTitle(0.6f);
weights.setSummary(0.2f);
weights.setContent(0.2f);
properties.setEmbeddingWeights(weights);

// 평가 수행 - UserMetrics 수집
// 평가 수행 - MMR bypass, 1차 후보군만
List<UserMetrics> userMetrics = testUsers.stream()
.map(user -> evaluateUserWithGroundTruth(user, properties))
.map(user -> evaluateUserCandidatesOnly(user, properties))
.filter(Optional::isPresent)
.map(Optional::get)
.toList();
Expand Down Expand Up @@ -116,6 +118,7 @@ private KMetrics calculateAverageKMetrics(List<UserMetrics> userMetrics) {
double n8 = userMetrics.stream().mapToDouble(UserMetrics::getNdcg8).average().orElse(0.0);
double r30 = userMetrics.stream().mapToDouble(UserMetrics::getRecall30).average().orElse(0.0);
double n30 = userMetrics.stream().mapToDouble(UserMetrics::getNdcg30).average().orElse(0.0);
double latency = userMetrics.stream().mapToDouble(UserMetrics::getLatencyMs).average().orElse(0.0);

return KMetrics.builder()
.recallAt4(r4)
Expand All @@ -124,17 +127,18 @@ private KMetrics calculateAverageKMetrics(List<UserMetrics> userMetrics) {
.ndcgAt8(n8)
.recallAt30(r30)
.ndcgAt30(n30)
.avgLatencyMs(latency)
.build();
}

private void printKComparisonHeader() {
log.info("");
log.info("설정 | K값 | Candidates | R@4 | R@8 | R@30 | nDCG@4 | nDCG@8 | nDCG@30 | 실행시간");
log.info("----------------------------------------------------------------------------------------------");
log.info("설정 | K값 | Candidates | R@4 | R@8 | R@30 | nDCG@4 | nDCG@8 | nDCG@30 | Latency | 실행시간");
log.info("-----------------------------------------------------------------------------------------------------------");
}

private void printKResult(KResult result) {
log.info(String.format("%-30s | %-9s | %-10s | %.4f | %.4f | %.4f | %.4f | %.4f | %.4f | %dms",
log.info(String.format("%-30s | %-9s | %-10s | %.4f | %.4f | %.4f | %.4f | %.4f | %.4f | %.0fms | %dms",
result.name,
result.knnSearchSize,
result.numCandidates,
Expand All @@ -144,6 +148,7 @@ private void printKResult(KResult result) {
result.metrics.ndcgAt4,
result.metrics.ndcgAt8,
result.metrics.ndcgAt30,
result.metrics.avgLatencyMs,
result.executionTimeMs
));
}
Expand Down Expand Up @@ -219,6 +224,25 @@ private void printBestKResult(List<KResult> results) {
});
}

private List<Map<String, Object>> toReportEntries(List<KResult> results) {
List<Map<String, Object>> entries = new ArrayList<>();
for (KResult r : results) {
Map<String, Object> entry = new LinkedHashMap<>();
entry.put("configName", r.name);
entry.put("knnSearchSize", r.knnSearchSize);
entry.put("numCandidates", r.numCandidates);
entry.put("averageRecall4", Math.round(r.metrics.recallAt4 * 10000.0) / 10000.0);
entry.put("averageRecall8", Math.round(r.metrics.recallAt8 * 10000.0) / 10000.0);
entry.put("averageRecall30", Math.round(r.metrics.recallAt30 * 10000.0) / 10000.0);
entry.put("averageNDCG4", Math.round(r.metrics.ndcgAt4 * 10000.0) / 10000.0);
entry.put("averageNDCG8", Math.round(r.metrics.ndcgAt8 * 10000.0) / 10000.0);
entry.put("averageNDCG30", Math.round(r.metrics.ndcgAt30 * 10000.0) / 10000.0);
entry.put("avgLatencyMs", Math.round(r.metrics.avgLatencyMs * 100.0) / 100.0);
entries.add(entry);
}
return entries;
}

@Getter
@Builder
private static class KConfig {
Expand All @@ -236,6 +260,7 @@ private static class KMetrics {
private double ndcgAt8;
private double recallAt30;
private double ndcgAt30;
private double avgLatencyMs;
}

@Getter
Expand Down
147 changes: 96 additions & 51 deletions src/test/java/com/techfork/evaluation/recommendation/LambdaOptimizationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,86 +1,131 @@
package com.techfork.evaluation.recommendation;

import com.techfork.domain.recommendation.config.RecommendationProperties;
import com.techfork.domain.user.entity.User;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.*;

/**
* MMR Lambda 파라미터 최적화 테스트
* MMR Lambda 파라미터 최적화 테스트 (Phase 4)
*
* Phase 1~3에서 결정된 최적값을 고정하고, lambda만 변화시켜 테스트.
* MMR Top-K 샘플링을 비활성화(결정적)하여 재현 가능한 평가.
*/
@Tag("evaluation")
@Slf4j
public class LambdaOptimizationTest extends RecommendationTestBase {

@Test
@DisplayName("Lambda 최적화 - Ground-Truth 기반 평가")
void optimizeLambdaWithGroundTruth() {
log.info("===== Lambda 최적화 테스트 (Ground-Truth 기반) =====");
private static final String REPORT_FILE = "evaluation-report-recommendation-phase4.json";

if (cachedGroundTruth == null || cachedGroundTruth.isEmpty()) {
log.warn("Ground-Truth 데이터가 없습니다. Fixture 로드를 확인하세요.");
return;
}
// Phase 1 최적 가중치
private static final float BEST_TITLE_WEIGHT = 0.6f;
private static final float BEST_SUMMARY_WEIGHT = 0.2f;
private static final float BEST_CONTENT_WEIGHT = 0.2f;

// Phase 2 최적 K값
private static final int BEST_KNN_SEARCH_SIZE = 50;
private static final int BEST_NUM_CANDIDATES = 150;

log.info("가중치 고정: 제목(0.5) + 요약(0.5)");
log.info("Lambda 범위: 0.0 ~ 1.0 (0.1 단위)");
// Phase 3 최적 후보군 크기
private static final int BEST_MMR_CANDIDATE_SIZE = 60;
private static final int BEST_MMR_FINAL_SIZE = 30;

@Test
@DisplayName("Lambda 최적화 - Phase 1~3 최적값 고정, 결정적 MMR")
void optimizeLambda() throws Exception {
log.info("===== Lambda 최적화 테스트 (Phase 4) =====");
log.info("Ground-Truth: {} 명 사용자", cachedGroundTruth.size());
log.info("고정값: title={}, summary={}, content={}, knnSearchSize={}, numCandidates={}, mmrCandidateSize={}",
BEST_TITLE_WEIGHT, BEST_SUMMARY_WEIGHT, BEST_CONTENT_WEIGHT,
BEST_KNN_SEARCH_SIZE, BEST_NUM_CANDIDATES, BEST_MMR_CANDIDATE_SIZE);

List<ConfigCombo> configs = createLambdaTestConfigs();
List<Double> lambdaValues = List.of(0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 1.0);
List<User> testUsers = getTestUsers();
log.info("테스트 사용자: {} 명", testUsers.size());
log.info("Lambda 범위: {}", lambdaValues);

printLambdaOptimizationHeader();
List<EvaluationResult> results = configs.stream()
.map(config -> {
EvaluationResult result = evaluateConfigWithGroundTruthAndILD(config, testUsers);
printLambdaOptimizationResult(result);
return result;
})
.toList();
printHeader();
List<EvaluationResult> results = evaluateAll(lambdaValues, testUsers);
printBestResult(results);

printBestLambdaResults(results);
saveRecommendationReport(REPORT_FILE, "Lambda 최적화", true, results);
}

private List<ConfigCombo> createLambdaTestConfigs() {
List<ConfigCombo> configs = new ArrayList<>();
// Lambda 0.0 ~ 1.0 (0.1 단위)
for (int i = 0; i <= 10; i++) {
double lambda = i / 10.0;
configs.add(ConfigCombo.builder()
.name(String.format("T0.5/S0.5 λ=%.1f", lambda))
.titleWeight(0.5f)
.summaryWeight(0.5f)
.contentWeight(0.0f)
.mmrLambda(lambda)
.build());
private List<EvaluationResult> evaluateAll(List<Double> lambdaValues, List<User> testUsers) {
List<EvaluationResult> results = new ArrayList<>();

for (Double lambda : lambdaValues) {
RecommendationProperties props = new RecommendationProperties();
props.setKnnSearchSize(BEST_KNN_SEARCH_SIZE);
props.setNumCandidates(BEST_NUM_CANDIDATES);
props.setMmrCandidateSize(BEST_MMR_CANDIDATE_SIZE);
props.setMmrFinalSize(BEST_MMR_FINAL_SIZE);
props.setLambda(lambda);
props.setMmrFirstTopK(1);
props.setMmrTopK(1);

RecommendationProperties.EmbeddingWeights weights = new RecommendationProperties.EmbeddingWeights();
weights.setTitle(BEST_TITLE_WEIGHT);
weights.setSummary(BEST_SUMMARY_WEIGHT);
weights.setContent(BEST_CONTENT_WEIGHT);
props.setEmbeddingWeights(weights);

String configName = String.format("λ=%.2f", lambda);

List<UserMetrics> metrics = testUsers.stream()
.map(user -> evaluateUserWithGroundTruthAndILD(user, props))
.filter(Optional::isPresent)
.map(Optional::get)
.toList();

EvaluationResult result = calculateAverageMetrics(configName, metrics);
results.add(result);
printResult(result);
}
return configs;

return results;
}

private void printHeader() {
log.info("");
log.info(String.format("%-12s | %-8s | %-8s | %-8s | %-8s | %-8s | %-8s | %-8s | %-10s | %-8s",
"설정", "R@4", "R@8", "R@30", "nDCG@4", "nDCG@8", "nDCG@30", "ILD", "Composite", "Latency"));
log.info("-".repeat(115));
}

private void printBestLambdaResults(List<EvaluationResult> results) {
log.info("\n===== Lambda 최적화 결과 요약 (K=8 기준) =====");
private void printResult(EvaluationResult result) {
log.info(String.format("%-12s | %.4f | %.4f | %.4f | %.4f | %.4f | %.4f | %.4f | %.4f | %.0fms",
result.getConfigName(),
result.getAvgRecall4(), result.getAvgRecall8(), result.getAvgRecall30(),
result.getAvgNdcg4(), result.getAvgNdcg8(), result.getAvgNdcg30(),
result.getAvgIld(), result.getCompositeScore(), result.getAvgLatencyMs()));
}

private void printBestResult(List<EvaluationResult> results) {
log.info("");
log.info("===== 최적 Lambda =====");

// 복합 점수 최고
results.stream()
.max(Comparator.comparingDouble(EvaluationResult::getCompositeScore))
.ifPresent(best -> log.info(String.format("[복합 점수 최고] %s | R@8: %.4f | nDCG@8: %.4f | ILD: %.4f | Score: %.4f",
best.getConfigName(), best.getAvgRecall8(), best.getAvgNdcg8(), best.getAvgIld(), best.getCompositeScore())));
.ifPresent(best -> log.info(String.format(
"[Composite 최고] %s | R@8: %.4f | nDCG@8: %.4f | ILD: %.4f | Score: %.4f | Latency: %.0fms",
best.getConfigName(), best.getAvgRecall8(), best.getAvgNdcg8(),
best.getAvgIld(), best.getCompositeScore(), best.getAvgLatencyMs())));

// 다양성(ILD) 최고
results.stream()
.max(Comparator.comparingDouble(EvaluationResult::getAvgIld))
.ifPresent(best -> log.info(String.format("[다양성(ILD) 최고] %s | ILD: %.4f",
best.getConfigName(), best.getAvgIld())));
.max(Comparator.comparingDouble(r -> (r.getAvgRecall8() + r.getAvgNdcg8()) / 2.0))
.ifPresent(best -> log.info(String.format(
"[정확성 최고 (R@8+nDCG@8)] %s | R@8: %.4f | nDCG@8: %.4f | ILD: %.4f",
best.getConfigName(), best.getAvgRecall8(), best.getAvgNdcg8(), best.getAvgIld())));

// Recall@8 최고
results.stream()
.max(Comparator.comparingDouble(EvaluationResult::getAvgRecall8))
.ifPresent(best -> log.info(String.format("[Recall@8 최고] %s | R@8: %.4f",
best.getConfigName(), best.getAvgRecall8())));
.max(Comparator.comparingDouble(EvaluationResult::getAvgIld))
.ifPresent(best -> log.info(String.format(
"[다양성 최고 (ILD)] %s | ILD: %.4f | R@8: %.4f | nDCG@8: %.4f",
best.getConfigName(), best.getAvgIld(), best.getAvgRecall8(), best.getAvgNdcg8())));
}
}
}
Loading