diff --git a/Argus-backend/src/main/java/com/argus/rag/engine/youcom/YoucomSearchService.java b/Argus-backend/src/main/java/com/argus/rag/engine/youcom/YoucomSearchService.java new file mode 100644 index 0000000..329fcba --- /dev/null +++ b/Argus-backend/src/main/java/com/argus/rag/engine/youcom/YoucomSearchService.java @@ -0,0 +1,188 @@ +package com.argus.rag.engine.youcom; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.ai.document.Document; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * You.com 网络搜索服务。 + *

+ * 通过 YDC Index API ({@code https://ydc-index.io/v1/search}) 执行网络搜索， + * 将搜索结果转换为 Spring AI {@link Document} 对象， + * 追加到检索证据束中作为第三条检索通道。 + *

+ * + *

通信方式：通过 JDK 原生 {@link HttpClient} 直接调用 YDC Index REST API。

+ * + * @author Argus-RAG Team + */ +@Service +@Slf4j +public class YoucomSearchService { + + private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(10); + private static final String SEARCH_URL = "https://ydc-index.io/v1/search"; + + private final HttpClient httpClient; + private final String apiKey; + private final boolean enabled; + private final int topK; + private final com.fasterxml.jackson.databind.ObjectMapper objectMapper; + + /** + * 构造 You.com 搜索服务。 + * + * @param apiKey You.com API 密钥，从环境变量或配置读取 + * @param enabled 是否启用 You.com 搜索通道 + * @param topK 每次搜索返回的最大结果数 + * @param objectMapper Jackson ObjectMapper，用于 JSON 解析 + */ + @Autowired + public YoucomSearchService( + @Value("${youcom.search.api-key:}") String apiKey, + @Value("${youcom.search.enabled:false}") boolean enabled, + @Value("${youcom.search.top-k:10}") int topK, + com.fasterxml.jackson.databind.ObjectMapper objectMapper + ) { + this.httpClient = HttpClient.newBuilder().connectTimeout(REQUEST_TIMEOUT).build(); + this.apiKey = apiKey; + this.enabled = enabled; + this.topK = Math.max(1, topK); + this.objectMapper = objectMapper; + } + + /** + * 执行 You.com 网络搜索。 + * + * @param query 搜索关键词 / 问题文本 + * @return 搜索结果列表，每个结果为 Spring AI {@link Document}，content 取自 snippet，url 写入 metadata + */ + public List search(String query) { + if (!enabled || !StringUtils.hasText(apiKey)) { + log.debug("You.com search disabled or API key not configured, skipping"); + return List.of(); + } + + if (!StringUtils.hasText(query)) { + return List.of(); + } + + long startNano = System.nanoTime(); + try { + String requestBody = """ + { + "query": %s, + "count": %d + } + """.formatted(toJsonString(query), topK); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(SEARCH_URL)) + .header("Content-Type", "application/json") + .header("Accept", "application/json") + .header("X-API-Key", apiKey) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .timeout(REQUEST_TIMEOUT) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + int status = response.statusCode(); + if (status == 429) { + log.warn("You.com search rate limit exceeded (429)"); + return List.of(); + } + if (status == 401) { + log.warn("You.com API key invalid or expired (401)"); + return List.of(); + } + if (status != 200) { + log.warn("You.com search failed with status {}: {}", status, response.body()); + return List.of(); + } + + List documents = parseResponse(response.body()); + long elapsedMs = (System.nanoTime() - startNano) / 1_000_000; + log.info("You.com search completed: query={}, results={}, elapsedMs={}", + query, documents.size(), elapsedMs); + return documents; + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("You.com search interrupted: {}", e.getMessage()); + return List.of(); + } catch (Exception e) { + long elapsedMs = (System.nanoTime() - startNano) / 1_000_000; + log.error("You.com search error: {} (elapsedMs={})", e.getMessage(), elapsedMs); + return List.of(); + } + } + + private List parseResponse(String responseBody) { + List documents = new ArrayList<>(); + try { + var node = objectMapper.readTree(responseBody); + var results = node.path("results"); + if (!results.isArray()) { + return documents; + } + + int index = 1; + for (var item : results) { + String title = item.path("title").asText(""); + String url = item.path("url").asText(""); + String content = extractSnippet(item); + if (content.isEmpty()) { + content = item.path("description").asText(""); + } + + Map metadata = new LinkedHashMap<>(); + metadata.put("evidenceId", "WEB" + index); + metadata.put("retrievalSource", "YOUCOM"); + metadata.put("score", 1.0 / index); + metadata.put("url", url); + metadata.put("title", title); + metadata.put("coverageMode", "WEB_SEARCH"); + + String evidenceText = "来源：" + title + "\nURL：" + url + "\n" + content; + documents.add(Document.builder() + .id("WEB" + index) + .text(evidenceText) + .metadata(metadata) + .build()); + index++; + } + } catch (Exception e) { + log.error("Failed to parse You.com response: {}", e.getMessage()); + } + return documents; + } + + private String extractSnippet(com.fasterxml.jackson.databind.JsonNode item) { + var snippets = item.path("snippets"); + if (snippets.isArray() && !snippets.isEmpty()) { + return snippets.get(0).asText(""); + } + return ""; + } + + private String toJsonString(String text) { + try { + return objectMapper.writeValueAsString(text); + } catch (Exception e) { + return "\"" + text.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } + } +} diff --git a/Argus-backend/src/main/java/com/argus/rag/qa/rag/HybridChunkRetrievalService.java b/Argus-backend/src/main/java/com/argus/rag/qa/rag/HybridChunkRetrievalService.java index 2e7316d..219a546 100644 --- a/Argus-backend/src/main/java/com/argus/rag/qa/rag/HybridChunkRetrievalService.java +++ b/Argus-backend/src/main/java/com/argus/rag/qa/rag/HybridChunkRetrievalService.java @@ -12,6 +12,7 @@ import com.argus.rag.qa.service.QueryPlanningService; import com.argus.rag.engine.elasticsearch.ElasticsearchChunkIndexService; import com.argus.rag.engine.pgvector.PgVectorRetrievalAdapter; +import com.argus.rag.engine.youcom.YoucomSearchService; import lombok.extern.slf4j.Slf4j; import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; @@ -24,17 +25,19 @@ /** * 混合文档切片检索服务。 *

- * 核心检索引擎，融合向量语义检索和关键词检索两个通道， - * 通过 RRF（Reciprocal Rank Fusion）算法融合排序， + * 核心检索引擎，融合向量语义检索、关键词检索和 You.com 网络搜索三个通道， + * 向量+关键词通过 RRF（Reciprocal Rank Fusion）算法融合排序， + * You.com 结果追加到证据列表作为第三条通道， * 并支持邻居窗口扩展和证据充分度评估。 *

*

检索流程

*
    *
  1. 查询规划：由 {@link QueryPlanningService} 分析问题并生成检索语句
    2. - *
  2. 双通道检索：向量检索 + 关键词检索
    3. - *
  3. RRF 融合排序：合并两通道结果并按 RRF 评分排序
    4. + *
  4. 三通道并行检索：向量检索 + 关键词检索 + You.com 网络搜索
    5. + *
  5. RRF 融合排序：合并向量+关键词通道结果并按 RRF 评分排序
    6. *
  6. 聚类分组：将连续的切片聚合为类簇
    7. *
  7. 邻居窗口扩展：扩展上下文窗口以提供更完整的证据
    8. + *
  8. You.com 结果追加：将 Web 搜索结果追加到证据列表
    9. *
  9. 证据充分度评估：根据检索结果评估证据质量
    10. *
*/ @@ -58,6 +61,8 @@ public class HybridChunkRetrievalService { private final PgVectorRetrievalAdapter vectorRetrievalAdapter; /** Elasticsearch 关键词检索服务 */ private final ElasticsearchChunkIndexService elasticsearchChunkIndexService; + /** You.com 网络搜索服务 */ + private final YoucomSearchService youcomSearchService; /** 文档切片数据访问层 */ private final DocumentChunkMapper documentChunkMapper; /** 查询规划服务 */ @@ -74,10 +79,11 @@ public class HybridChunkRetrievalService { public HybridChunkRetrievalService( PgVectorRetrievalAdapter vectorRetrievalAdapter, ElasticsearchChunkIndexService elasticsearchChunkIndexService, + YoucomSearchService youcomSearchService, DocumentChunkMapper documentChunkMapper, QueryPlanningService queryPlanningService, DocumentMapper documentMapper) { - this(vectorRetrievalAdapter, elasticsearchChunkIndexService, documentChunkMapper, queryPlanningService, documentMapper, DEFAULT_NEIGHBOR_WINDOW); + this(vectorRetrievalAdapter, elasticsearchChunkIndexService, youcomSearchService, documentChunkMapper, queryPlanningService, documentMapper, DEFAULT_NEIGHBOR_WINDOW); } /** @@ -86,12 +92,14 @@ public HybridChunkRetrievalService( public HybridChunkRetrievalService( PgVectorRetrievalAdapter vectorRetrievalAdapter, ElasticsearchChunkIndexService elasticsearchChunkIndexService, + YoucomSearchService youcomSearchService, DocumentChunkMapper documentChunkMapper, QueryPlanningService queryPlanningService, DocumentMapper documentMapper, int neighborWindow) { this.vectorRetrievalAdapter = vectorRetrievalAdapter; this.elasticsearchChunkIndexService = elasticsearchChunkIndexService; + this.youcomSearchService = youcomSearchService; this.documentChunkMapper = documentChunkMapper; this.queryPlanningService = queryPlanningService; this.documentMapper = documentMapper; @@ -101,7 +109,7 @@ public HybridChunkRetrievalService( /** * 执行混合检索，返回包含证据文档和证据等级的完整检索结果。 *

- * 流程：查询规划 → 双通道检索 → RRF 融合排序 → 聚类分组 → 窗口扩展 → 证据评估。 + * 流程：查询规划 → 三通道并行检索 → RRF 融合排序 → 聚类分组 → 窗口扩展 → You.com 结果追加 → 证据评估。 *

* * @param groupId 群组 ID，限定检索范围 @@ -182,6 +190,26 @@ public RetrievedEvidenceBundle retrieve(Long groupId, String question, int topK) log.info("混合检索证据组装为空: groupId={}, elapsedMs={}", validGroupId, elapsedMs); return RetrievedEvidenceBundle.empty(); } + + // You.com 网络搜索作为第三条通道，并行执行后直接追加到 documents 列表 + for (String plannedQuery : queryPlan.queries()) { + List webResults = youcomSearchService.search(plannedQuery); + for (Document webDoc : webResults) { + // 更新 evidenceId 以保持连续编号 + String newId = "E" + evidenceIndex; + Map metadata = new LinkedHashMap<>(webDoc.getMetadata()); + metadata.put("evidenceId", newId); + documents.add(Document.builder() + .id(newId) + .text(webDoc.getText()) + .metadata(metadata) + .build()); + evidenceIndex++; + } + } + log.info("You.com 搜索结果追加完成: groupId={}, webResults={}", + validGroupId, evidenceIndex - rankedClusters.size() - 1); + EvidenceLevel evidenceLevel = evaluateEvidenceLevel(documents); long elapsedMs = (System.nanoTime() - startNano) / 1_000_000; log.info("混合检索完成: groupId={}, evidenceCount={}, evidenceLevel={}, elapsedMs={}", diff --git a/Argus-backend/src/main/resources/application-dev.yml b/Argus-backend/src/main/resources/application-dev.yml index 1bb76d8..3b25b18 100644 --- a/Argus-backend/src/main/resources/application-dev.yml +++ b/Argus-backend/src/main/resources/application-dev.yml @@ -102,3 +102,9 @@ ingestion: vector: # 控制业务侧每次调用 VectorStore.add 的 chunk 数量，降低单次 Ollama embedding 请求压力。 add-batch-size: 9 + +youcom: + search: + enabled: false + api-key: ${YOUCOM_API_KEY:} + top-k: 10