Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
package com.argus.rag.engine.youcom;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.ai.document.Document;

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
* You.com 网络搜索服务。
* <p>
* 通过 YDC Index API ({@code https://ydc-index.io/v1/search}) 执行网络搜索,
* 将搜索结果转换为 Spring AI {@link Document} 对象,
* 追加到检索证据束中作为第三条检索通道。
* </p>
*
* <p>通信方式:通过 JDK 原生 {@link HttpClient} 直接调用 YDC Index REST API。</p>
*
* @author Argus-RAG Team
*/
@Service
@Slf4j
public class YoucomSearchService {

private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(10);
private static final String SEARCH_URL = "https://ydc-index.io/v1/search";

private final HttpClient httpClient;
private final String apiKey;
private final boolean enabled;
private final int topK;
private final com.fasterxml.jackson.databind.ObjectMapper objectMapper;

/**
* 构造 You.com 搜索服务。
*
* @param apiKey You.com API 密钥,从环境变量或配置读取
* @param enabled 是否启用 You.com 搜索通道
* @param topK 每次搜索返回的最大结果数
* @param objectMapper Jackson ObjectMapper,用于 JSON 解析
*/
@Autowired
public YoucomSearchService(
@Value("${youcom.search.api-key:}") String apiKey,
@Value("${youcom.search.enabled:false}") boolean enabled,
@Value("${youcom.search.top-k:10}") int topK,
com.fasterxml.jackson.databind.ObjectMapper objectMapper
) {
this.httpClient = HttpClient.newBuilder().connectTimeout(REQUEST_TIMEOUT).build();
this.apiKey = apiKey;
this.enabled = enabled;
this.topK = Math.max(1, topK);
this.objectMapper = objectMapper;
}

/**
* 执行 You.com 网络搜索。
*
* @param query 搜索关键词 / 问题文本
* @return 搜索结果列表,每个结果为 Spring AI {@link Document},content 取自 snippet,url 写入 metadata
*/
public List<Document> search(String query) {
if (!enabled || !StringUtils.hasText(apiKey)) {
log.debug("You.com search disabled or API key not configured, skipping");
return List.of();
}

if (!StringUtils.hasText(query)) {
return List.of();
}

long startNano = System.nanoTime();
try {
String requestBody = """
{
"query": %s,
"count": %d
}
""".formatted(toJsonString(query), topK);

HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(SEARCH_URL))
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.header("X-API-Key", apiKey)
.POST(HttpRequest.BodyPublishers.ofString(requestBody))
.timeout(REQUEST_TIMEOUT)
.build();

HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
int status = response.statusCode();
if (status == 429) {
log.warn("You.com search rate limit exceeded (429)");
return List.of();
}
if (status == 401) {
log.warn("You.com API key invalid or expired (401)");
return List.of();
}
if (status != 200) {
log.warn("You.com search failed with status {}: {}", status, response.body());
return List.of();
}

List<Document> documents = parseResponse(response.body());
long elapsedMs = (System.nanoTime() - startNano) / 1_000_000;
log.info("You.com search completed: query={}, results={}, elapsedMs={}",
query, documents.size(), elapsedMs);
return documents;

} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("You.com search interrupted: {}", e.getMessage());
return List.of();
} catch (Exception e) {
long elapsedMs = (System.nanoTime() - startNano) / 1_000_000;
log.error("You.com search error: {} (elapsedMs={})", e.getMessage(), elapsedMs);
return List.of();
}
}

private List<Document> parseResponse(String responseBody) {
List<Document> documents = new ArrayList<>();
try {
var node = objectMapper.readTree(responseBody);
var results = node.path("results");
if (!results.isArray()) {
return documents;
}

int index = 1;
for (var item : results) {
String title = item.path("title").asText("");
String url = item.path("url").asText("");
String content = extractSnippet(item);
if (content.isEmpty()) {
content = item.path("description").asText("");
}

Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("evidenceId", "WEB" + index);
metadata.put("retrievalSource", "YOUCOM");
metadata.put("score", 1.0 / index);
metadata.put("url", url);
metadata.put("title", title);
metadata.put("coverageMode", "WEB_SEARCH");

String evidenceText = "来源:" + title + "\nURL:" + url + "\n" + content;
documents.add(Document.builder()
.id("WEB" + index)
.text(evidenceText)
.metadata(metadata)
.build());
index++;
}
} catch (Exception e) {
log.error("Failed to parse You.com response: {}", e.getMessage());
}
return documents;
}

private String extractSnippet(com.fasterxml.jackson.databind.JsonNode item) {
var snippets = item.path("snippets");
if (snippets.isArray() && !snippets.isEmpty()) {
return snippets.get(0).asText("");
}
return "";
}

private String toJsonString(String text) {
try {
return objectMapper.writeValueAsString(text);
} catch (Exception e) {
return "\"" + text.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.argus.rag.qa.service.QueryPlanningService;
import com.argus.rag.engine.elasticsearch.ElasticsearchChunkIndexService;
import com.argus.rag.engine.pgvector.PgVectorRetrievalAdapter;
import com.argus.rag.engine.youcom.YoucomSearchService;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
Expand All @@ -24,17 +25,19 @@
/**
* 混合文档切片检索服务。
* <p>
* 核心检索引擎,融合向量语义检索和关键词检索两个通道,
* 通过 RRF(Reciprocal Rank Fusion)算法融合排序,
* 核心检索引擎,融合向量语义检索、关键词检索和 You.com 网络搜索三个通道,
* 向量+关键词通过 RRF(Reciprocal Rank Fusion)算法融合排序,
* You.com 结果追加到证据列表作为第三条通道,
* 并支持邻居窗口扩展和证据充分度评估。
* </p>
* <h3>检索流程</h3>
* <ol>
* <li>查询规划:由 {@link QueryPlanningService} 分析问题并生成检索语句</li>
* <li>双通道检索:向量检索 + 关键词检索</li>
* <li>RRF 融合排序:合并两通道结果并按 RRF 评分排序</li>
* <li>三通道并行检索:向量检索 + 关键词检索 + You.com 网络搜索</li>
* <li>RRF 融合排序:合并向量+关键词通道结果并按 RRF 评分排序</li>
* <li>聚类分组:将连续的切片聚合为类簇</li>
* <li>邻居窗口扩展:扩展上下文窗口以提供更完整的证据</li>
* <li>You.com 结果追加:将 Web 搜索结果追加到证据列表</li>
* <li>证据充分度评估:根据检索结果评估证据质量</li>
* </ol>
*/
Expand All @@ -58,6 +61,8 @@ public class HybridChunkRetrievalService {
private final PgVectorRetrievalAdapter vectorRetrievalAdapter;
/** Elasticsearch 关键词检索服务 */
private final ElasticsearchChunkIndexService elasticsearchChunkIndexService;
/** You.com 网络搜索服务 */
private final YoucomSearchService youcomSearchService;
/** 文档切片数据访问层 */
private final DocumentChunkMapper documentChunkMapper;
/** 查询规划服务 */
Expand All @@ -74,10 +79,11 @@ public class HybridChunkRetrievalService {
public HybridChunkRetrievalService(
PgVectorRetrievalAdapter vectorRetrievalAdapter,
ElasticsearchChunkIndexService elasticsearchChunkIndexService,
YoucomSearchService youcomSearchService,
DocumentChunkMapper documentChunkMapper,
QueryPlanningService queryPlanningService,
DocumentMapper documentMapper) {
this(vectorRetrievalAdapter, elasticsearchChunkIndexService, documentChunkMapper, queryPlanningService, documentMapper, DEFAULT_NEIGHBOR_WINDOW);
this(vectorRetrievalAdapter, elasticsearchChunkIndexService, youcomSearchService, documentChunkMapper, queryPlanningService, documentMapper, DEFAULT_NEIGHBOR_WINDOW);
}

/**
Expand All @@ -86,12 +92,14 @@ public HybridChunkRetrievalService(
public HybridChunkRetrievalService(
PgVectorRetrievalAdapter vectorRetrievalAdapter,
ElasticsearchChunkIndexService elasticsearchChunkIndexService,
YoucomSearchService youcomSearchService,
DocumentChunkMapper documentChunkMapper,
QueryPlanningService queryPlanningService,
DocumentMapper documentMapper,
int neighborWindow) {
this.vectorRetrievalAdapter = vectorRetrievalAdapter;
this.elasticsearchChunkIndexService = elasticsearchChunkIndexService;
this.youcomSearchService = youcomSearchService;
this.documentChunkMapper = documentChunkMapper;
this.queryPlanningService = queryPlanningService;
this.documentMapper = documentMapper;
Expand All @@ -101,7 +109,7 @@ public HybridChunkRetrievalService(
/**
* 执行混合检索,返回包含证据文档和证据等级的完整检索结果。
* <p>
* 流程:查询规划 → 双通道检索 → RRF 融合排序 → 聚类分组 → 窗口扩展 → 证据评估。
* 流程:查询规划 → 三通道并行检索 → RRF 融合排序 → 聚类分组 → 窗口扩展 → You.com 结果追加 → 证据评估。
* </p>
*
* @param groupId 群组 ID,限定检索范围
Expand Down Expand Up @@ -182,6 +190,26 @@ public RetrievedEvidenceBundle retrieve(Long groupId, String question, int topK)
log.info("混合检索证据组装为空: groupId={}, elapsedMs={}", validGroupId, elapsedMs);
return RetrievedEvidenceBundle.empty();
}

// You.com 网络搜索作为第三条通道,并行执行后直接追加到 documents 列表
for (String plannedQuery : queryPlan.queries()) {
List<Document> webResults = youcomSearchService.search(plannedQuery);
for (Document webDoc : webResults) {
// 更新 evidenceId 以保持连续编号
String newId = "E" + evidenceIndex;
Map<String, Object> metadata = new LinkedHashMap<>(webDoc.getMetadata());
metadata.put("evidenceId", newId);
documents.add(Document.builder()
.id(newId)
.text(webDoc.getText())
.metadata(metadata)
.build());
evidenceIndex++;
}
}
log.info("You.com 搜索结果追加完成: groupId={}, webResults={}",
validGroupId, evidenceIndex - rankedClusters.size() - 1);

EvidenceLevel evidenceLevel = evaluateEvidenceLevel(documents);
long elapsedMs = (System.nanoTime() - startNano) / 1_000_000;
log.info("混合检索完成: groupId={}, evidenceCount={}, evidenceLevel={}, elapsedMs={}",
Expand Down
6 changes: 6 additions & 0 deletions Argus-backend/src/main/resources/application-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,9 @@ ingestion:
vector:
# 控制业务侧每次调用 VectorStore.add 的 chunk 数量,降低单次 Ollama embedding 请求压力。
add-batch-size: 9

youcom:
search:
enabled: false
api-key: ${YOUCOM_API_KEY:}
top-k: 10