From f1b9c5964f31913d94955b94a4df9654a6596169 Mon Sep 17 00:00:00 2001 From: lloydzhou Date: Wed, 17 Jun 2026 02:19:56 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20Read=20=E5=B7=A5=E5=85=B7=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E9=9D=9E=20UTF-8=20=E7=BC=96=E7=A0=81=E6=96=87?= =?UTF-8?q?=E4=BB=B6=EF=BC=88GBK/GB18030=20=E7=AD=89=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 读取 GBK 等非 UTF-8 编码文件时，原始字节直接进入 conversation 并提交 LLM API，导致请求报错。四个实现版本的 Read 工具均缺失编码处理。 修复方案（四版本统一）： - 检测文件编码（file -bi），非 UTF-8 时用 iconv 转码（GB18030 兼容 GBK） - 转码失败 fallback 到 sanitize_utf8（保证 JSON 序列化不破坏） - bash: 新增 util_iconv_if_needed 函数，tool_read 调用之 - go: toolRead 增加 utf8.Valid 检测 + iconvToUTF8 辅助函数 - rust: read() 改用 fs::read 读字节 + iconv_to_utf8 方法 - c: 新增 util_iconv_if_needed，tool_read 读取后调用 新增测试: test_read_gbk_encoding 验证 GBK 文件转码为 UTF-8 --- c/tools.c | 8 +++- c/util.c | 112 ++++++++++++++++++++++++++++++++++++++++++++++ c/util.h | 4 ++ go/tools.go | 55 ++++++++++++++++++++++- rust/src/tools.rs | 60 +++++++++++++++++++++++-- src/agent.sh | 35 ++++++++++++++- tests/test.sh | 27 +++++++++++ 7 files changed, 293 insertions(+), 8 deletions(-) diff --git a/c/tools.c b/c/tools.c index 8a76440..237eca4 100644 --- a/c/tools.c +++ b/c/tools.c @@ -84,8 +84,8 @@ static ToolResult tool_read(const char *input_json, int max_bytes) { int limit = json_get_int(jp.val, "limit"); /* 读取文件 */ - char *content = util_read_file(path); - if (!content) { + char *raw = util_read_file(path); + if (!raw) { StrBuf buf; sb_init(&buf); sb_appendf(&buf, "Error: file not found: %s", path); @@ -95,6 +95,10 @@ static ToolResult tool_read(const char *input_json, int max_bytes) { return r; } + /* 编码处理：非 UTF-8（GBK 等）自动 iconv 转码，失败 sanitize 兜底 */ + char *content = util_iconv_if_needed(raw); + free(raw); + /* 按行分割，添加行号 */ StrBuf buf; sb_init(&buf); diff --git a/c/util.c b/c/util.c index faefb99..b3d4f6e 100644 --- a/c/util.c +++ b/c/util.c @@ -177,6 +177,118 @@ char *util_sanitize_utf8(const char *src) { return sb.data; } +/* 检查字符串是否为合法 UTF-8。返回 1=合法, 0=非法 */ +static int is_valid_utf8(const char *src, size_t len) { + const unsigned char *p = (const unsigned char *)src; + const unsigned char *end = p + len; + while (p < end) { + unsigned char b = *p; + if (b < 0x80) { p++; } + else if (b >= 0xC2 && b <= 0xDF) { + if (p + 1 >= end || p[1] < 0x80 || p[1] > 0xBF) return 0; + p += 2; + } else if (b >= 0xE0 && b <= 0xEF) { + if (p + 2 >= end || p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) return 0; + p += 3; + } else if (b >= 0xF0 && b <= 0xF4) { + if (p + 3 >= end || p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF || p[3] < 0x80 || p[3] > 0xBF) return 0; + p += 4; + } else { + return 0; /* C0-C1, 80-BF, F5-FF */ + } + } + return 1; +} + +/* 用 popen 调用 file -bi 检测编码，返回检测到的 from_enc（malloc'd）。 + * iso-8859-* 一律视为 gb18030（兼容 GBK）。失败返回 NULL。 */ +static char *detect_encoding(const char *content, size_t len) { + char tmppath[256]; + snprintf(tmppath, sizeof(tmppath), "/tmp/iconv_detect_%d.tmp", (int)getpid()); + FILE *f = fopen(tmppath, "wb"); + if (!f) return NULL; + fwrite(content, 1, len, f); + fclose(f); + + char cmd[512]; + snprintf(cmd, sizeof(cmd), "file -bi '%s' 2>/dev/null", tmppath); + FILE *pp = popen(cmd, "r"); + if (!pp) { unlink(tmppath); return NULL; } + + char mime[256] = {0}; + if (fgets(mime, sizeof(mime), pp)) { + /* 转小写 */ + for (char *c = mime; *c; c++) { if (*c >= 'A' && *c <= 'Z') *c += 32; } + } + pclose(pp); + unlink(tmppath); + + /* 检查 charset */ + if (strstr(mime, "charset=utf-8") || strstr(mime, "charset=us-ascii")) { + return util_strdup("utf-8"); + } + /* 匹配已知中文/日文编码 */ + const char *encs[] = {"gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr", NULL}; + for (int i = 0; encs[i]; i++) { + char needle[32]; + snprintf(needle, sizeof(needle), "charset=%s", encs[i]); + if (strstr(mime, needle)) return util_strdup(encs[i]); + } + /* iso-8859-* → 视为 gb18030（file 对中文文件常误报 iso-8859-1） */ + if (strstr(mime, "charset=iso-8859")) return util_strdup("gb18030"); + return NULL; +} + +char *util_iconv_if_needed(const char *content) { + if (!content) return util_sanitize_utf8(""); + size_t len = strlen(content); + + /* UTF-8 合法 → 直接 sanitize（idempotent） */ + if (is_valid_utf8(content, len)) { + return util_sanitize_utf8(content); + } + + /* 非 UTF-8 → 尝试 iconv 转码 */ + char *from_enc = detect_encoding(content, len); + if (!from_enc) from_enc = util_strdup("gb18030"); /* 默认按 gb18030 尝试 */ + + /* 构建 iconv 管道：echo content | iconv -f gb18030 -t UTF-8//IGNORE */ + char tmppath[256]; + snprintf(tmppath, sizeof(tmppath), "/tmp/iconv_in_%d.tmp", (int)getpid()); + FILE *f = fopen(tmppath, "wb"); + if (!f) { free(from_enc); return util_sanitize_utf8(content); } + fwrite(content, 1, len, f); + fclose(f); + + char cmd[512]; + snprintf(cmd, sizeof(cmd), "iconv -f '%s' -t UTF-8//IGNORE '%s' 2>/dev/null", from_enc, tmppath); + free(from_enc); + + FILE *pp = popen(cmd, "r"); + if (!pp) { unlink(tmppath); return util_sanitize_utf8(content); } + + StrBuf sb; + sb_init(&sb); + char buf2[4096]; + size_t total = 0; + while (fgets(buf2, sizeof(buf2), pp)) { + sb_append(&sb, buf2); + total += strlen(buf2); + } + pclose(pp); + unlink(tmppath); + + if (total == 0 && len > 0) { + /* iconv 失败 → 兜底 */ + sb_free(&sb); + return util_sanitize_utf8(content); + } + + char *result = util_sanitize_utf8(sb.data); + sb_free(&sb); + return result; +} + /* ============================================================ * 工具函数 * ============================================================ */ diff --git a/c/util.h b/c/util.h index f191c97..60b727b 100644 --- a/c/util.h +++ b/c/util.h @@ -75,6 +75,10 @@ void util_truncate_chars(char *s, int max_chars); /* UTF-8 sanitize：将非法字节替换为 \ufffd 字面文本，返回新 malloc'd 字符串 */ char *util_sanitize_utf8(const char *src); +/* 编码检测+转换：若 content 非 UTF-8（如 GBK），调用系统 iconv 转码后 sanitize。 + * 转码失败则 fallback 到 sanitize_utf8（保证 JSON 安全）。返回新 malloc'd 字符串。 */ +char *util_iconv_if_needed(const char *content); + /* trim 尾部空白 */ char *util_rtrim(char *s); diff --git a/go/tools.go b/go/tools.go index 7c48de4..8d11aca 100644 --- a/go/tools.go +++ b/go/tools.go @@ -15,6 +15,7 @@ import ( "strings" "syscall" "time" + "unicode/utf8" ) const defaultToolResultMaxBytes = 100000 @@ -188,6 +189,7 @@ func (td *ToolDispatcher) CallSummary(name string, params map[string]string) str // ─── 工具实现 ─── // toolRead 读取文件（支持 offset/limit） +// 非 UTF-8 编码（GBK 等）文件自动调用 iconv 转码，失败则 sanitize 兜底 func (td *ToolDispatcher) toolRead(p, offsetStr, limitStr string) (string, error) { if p == "" { return "", fmt.Errorf("no path provided") @@ -201,7 +203,20 @@ func (td *ToolDispatcher) toolRead(p, offsetStr, limitStr string) (string, error return "", fmt.Errorf("read error: %v", err) } - lines := strings.Split(string(data), "\n") + // 编码处理：UTF-8 合法 → 直接用；否则尝试 iconv 转码，失败 fallback sanitize + var content string + if utf8.Valid(data) { + content = sanitizeUTF8(data) + } else { + converted, ok := iconvToUTF8(data) + if ok { + content = converted + } else { + content = sanitizeUTF8(data) // 兜底：保证 JSON 序列化不破坏 + } + } + + lines := strings.Split(content, "\n") // offset 默认 1 offset := 1 if offsetStr != "" { @@ -746,6 +761,44 @@ func (td *ToolDispatcher) toolSubAgent(ctx context.Context, prompt, description, return "", fmt.Errorf("sub-agent launcher not configured") } +// ─── iconvToUTF8: 调用系统 iconv 将非 UTF-8 编码（GBK/GB18030 等）转为 UTF-8 ─── +// file 命令对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集。 +// 返回 (sanitized 内容, 是否成功)。失败时 ok=false，调用方应 fallback 到 sanitizeUTF8。 +func iconvToUTF8(data []byte) (string, bool) { + // 检测编码：优先用 file -bi，iso-8859-* 一律按 gb18030 处理（兼容 GBK） + fromEnc := "gb18030" + if tmpfile, err := os.CreateTemp("", "iconv-*.txt"); err == nil { + tmpfile.Write(data) + tmpfile.Close() + defer os.Remove(tmpfile.Name()) + if out, err := exec.Command("file", "-bi", tmpfile.Name()).Output(); err == nil { + mime := strings.ToLower(string(out)) + if strings.Contains(mime, "charset=utf-8") || strings.Contains(mime, "charset=us-ascii") { + return sanitizeUTF8(data), true // 实际是合法 UTF-8/ASCII + } + // 提取 charset，若明确非 iso-8859 则用检测到的值 + for _, enc := range []string{"gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"} { + if strings.Contains(mime, "charset="+enc) { + fromEnc = enc + break + } + } + } + } + // 调用 iconv 转码 + cmd := exec.Command("iconv", "-f", fromEnc, "-t", "UTF-8//IGNORE") + cmd.Stdin = bytes.NewReader(data) + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return "", false + } + if out.Len() == 0 && len(data) > 0 { + return "", false + } + return sanitizeUTF8(out.Bytes()), true +} + // ─── sanitizeUTF8: 过滤非法 UTF-8 字节，替换为字面 \ufffd ─── // 移植自 src/awk/sanitize_utf8.awk，确保 JSON 序列化不会因非法字节失败。 func sanitizeUTF8(data []byte) string { diff --git a/rust/src/tools.rs b/rust/src/tools.rs index 6662888..214570b 100644 --- a/rust/src/tools.rs +++ b/rust/src/tools.rs @@ -170,13 +170,23 @@ use crate::config::Config; if path.is_empty() { bail!("no path provided"); } - let data = fs::read_to_string(path) + let data = fs::read(path) .map_err(|_| anyhow!("Error: file not found or unreadable: {path}"))?; + // 编码处理：UTF-8 合法 → 直接用；否则尝试 iconv 转码，失败 fallback sanitize + let content = match std::str::from_utf8(&data) { + Ok(s) => self.sanitize_utf8(data.as_slice()), + Err(_) => { + match self.iconv_to_utf8(&data) { + Some(converted) => converted, + None => self.sanitize_utf8(data.as_slice()), + } + } + }; if offset.is_none() && limit.is_none() { - return Ok(data); + return Ok(content); } - let mut lines: Vec<&str> = data.split('\n').collect(); + let mut lines: Vec<&str> = content.split('\n').collect(); if !lines.is_empty() && lines.last().map(|l| l.is_empty()).unwrap_or(false) { lines.pop(); } @@ -1101,6 +1111,50 @@ use crate::config::Config; out } + /// iconv_to_utf8: 调用系统 iconv 将非 UTF-8 编码（GBK/GB18030 等）转为 UTF-8。 + /// file 命令对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集。 + /// 返回 Some(sanitized_content) 或 None（失败时调用方应 fallback 到 sanitize_utf8）。 + fn iconv_to_utf8(&self, data: &[u8]) -> Option { + // 检测编码：file -bi，iso-8859-* 一律按 gb18030 处理（兼容 GBK） + let mut from_enc = "gb18030"; + // 写临时文件用 file 检测 + let tmp = std::env::temp_dir().join(format!("iconv-{}.tmp", std::process::id())); + if fs::write(&tmp, data).is_ok() { + if let Ok(out) = Command::new("file").arg("-bi").arg(&tmp).output() { + let mime = String::from_utf8_lossy(&out.stdout).to_lowercase(); + if mime.contains("charset=utf-8") || mime.contains("charset=us-ascii") { + let _ = fs::remove_file(&tmp); + return Some(self.sanitize_utf8(data)); + } + for enc in &["gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"] { + if mime.contains(&format!("charset={enc}")) { + from_enc = enc; + break; + } + } + } + let _ = fs::remove_file(&tmp); + } + // 调用 iconv 转码 + let child = Command::new("iconv") + .arg("-f").arg(from_enc) + .arg("-t").arg("UTF-8//IGNORE") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + use std::io::Write; + let mut stdin = child.stdin?; + stdin.write_all(data).ok()?; + drop(stdin); + let output = child.wait_with_output().ok()?; + if output.stdout.is_empty() && !data.is_empty() { + return None; + } + Some(self.sanitize_utf8(&output.stdout)) + } + /// sanitize_utf8: replace illegal UTF-8 bytes with literal `\ufffd` text. /// Ported from src/awk/sanitize_utf8.awk — ensures JSON serialization won't /// fail on illegal bytes and produces the same `\ufffd` text the mock server checks for. diff --git a/src/agent.sh b/src/agent.sh index e07a768..be34a5f 100755 --- a/src/agent.sh +++ b/src/agent.sh @@ -892,15 +892,46 @@ tool_bash_mode_allows() { (( (8#$required & (4095 ^ 8#$allowed)) == 0 )) } +util_iconv_if_needed() { + # 读 stdin，若检测到非 UTF-8 编码则 iconv 转码，再 sanitize。 + # 常见中文编码: iso-8859-1/iso-8859-x 在 file 命令里可能报 iso-8859，但实际是 GBK/GB18030。 + local tmp; tmp=$(mktemp) + cat > "$tmp" + local enc="" + if command -v file >/dev/null 2>&1; then + enc=$(file -bi "$tmp" 2>/dev/null | sed -n 's/.*charset=\([^[:space:];]*\).*/\1/p') + fi + enc="${enc:-unknown}" + enc=$(printf '%s' "$enc" | tr 'A-Z' 'a-z') + # 已是 UTF-8 / ascii / unknown → 直接走 sanitize（idempotent，UTF-8 不受影响） + if [[ "$enc" == "utf-8" || "$enc" == "us-ascii" || "$enc" == "unknown" ]]; then + util_awk_run -f "$AWK_DIR/sanitize_utf8.awk" "$tmp" + else + # 非 UTF-8 编码（gbk/gb18030/iso-8859-x 等）→ 先 iconv 转码 + # file 对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集，优先用 + local from_enc="$enc" + if [[ "$enc" == iso-8859* ]]; then + from_enc="gb18030" # GB18030 兼容 GBK 和绝大多数中文编码 + fi + if iconv -f "$from_enc" -t UTF-8//IGNORE "$tmp" 2>/dev/null | util_awk_run -f "$AWK_DIR/sanitize_utf8.awk"; then + : # success + else + # iconv 失败 → 最后兜底：直接 sanitize 原始字节 + util_awk_run -f "$AWK_DIR/sanitize_utf8.awk" "$tmp" + fi + fi + rm -f "$tmp" +} + tool_read() { local path="$1" offset="${2:-1}" limit="${3:-0}" [[ -z "$path" ]] && { echo "no path provided"; return 1; } [[ ! -f "$path" ]] && { echo "file not found: $path"; return 1; } [[ ! -r "$path" ]] && { echo "permission denied: $path"; return 1; } if (( limit > 0 )); then - sed -n "${offset},$((offset + limit - 1))p" "$path" + sed -n "${offset},$((offset + limit - 1))p" "$path" | util_iconv_if_needed else - sed -n "${offset},\$p" "$path" + sed -n "${offset},\$p" "$path" | util_iconv_if_needed fi } diff --git a/tests/test.sh b/tests/test.sh index 352485d..8117bfb 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1040,6 +1040,32 @@ test_agent_bash_utf8_sanitize() { fi } +test_read_gbk_encoding() { + info "Test 25d: Read tool GBK encoding → iconv convert to UTF-8" + # 测试 iconv + sanitize 管道能正确转码 GBK 文件（util_iconv_if_needed 的核心逻辑） + local target_file output expected + target_file="/tmp/bash-agent-read-gbk-test.txt" + # 写入 GBK 编码内容: "hello 你好 world" 的 GBK 字节 + printf 'hello \xc4\xe3\xba\xc3 world\n' > "$target_file" + + # 验证 file 检测 + iconv 转码管道 + local enc from_enc + enc=$(file -bi "$target_file" 2>/dev/null | sed -n 's/.*charset=\([^[:space:];]*\).*/\1/p' | tr 'A-Z' 'a-z') + if [[ "$enc" == iso-8859* ]]; then + from_enc="gb18030" + else + from_enc="$enc" + fi + output=$(iconv -f "$from_enc" -t UTF-8//IGNORE "$target_file" 2>/dev/null | LC_ALL=C awk -f "$AWK_DIR/sanitize_utf8.awk") + + if echo "$output" | grep -q "你好" && ! echo "$output" | grep -q '\\ufffd'; then + green "Read tool GBK encoding"; ((PASS++)) || true + else + red "Read tool GBK encoding"; echo " enc=$enc from_enc=$from_enc output=$output"; ((FAIL++)) || true + fi + rm -f "$target_file" +} + test_agent_stream_tool_call() { info "Test 26: Agent.sh stream-json tool call" local output @@ -2534,6 +2560,7 @@ test_agent_bash_custom_mode_allows_system_read test_agent_bash_allows_dev_null_redirection test_agent_bash_long_result test_agent_bash_utf8_sanitize +test_read_gbk_encoding test_agent_stream_tool_call test_agent_stream_usage_event test_agent_tool_result_multiline_url