Skip to content

fix: Read 工具支持非 UTF-8 编码文件（GBK/GB18030 等） #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lloydzhou wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix: Read 工具支持非 UTF-8 编码文件（GBK/GB18030 等） #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions c/tools.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ static ToolResult tool_read(const char *input_json, int max_bytes) {
int limit = json_get_int(jp.val, "limit");

/* 读取文件 */
char *content = util_read_file(path);
if (!content) {
char *raw = util_read_file(path);
if (!raw) {
StrBuf buf;
sb_init(&buf);
sb_appendf(&buf, "Error: file not found: %s", path);
Expand All @@ -95,6 +95,10 @@ static ToolResult tool_read(const char *input_json, int max_bytes) {
return r;
}

/* 编码处理：非 UTF-8（GBK 等）自动 iconv 转码，失败 sanitize 兜底 */
char *content = util_iconv_if_needed(raw);
free(raw);

/* 按行分割，添加行号 */
StrBuf buf;
sb_init(&buf);
Expand Down
112 changes: 112 additions & 0 deletions c/util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,118 @@ char *util_sanitize_utf8(const char *src) {
return sb.data;
}

/* 检查字符串是否为合法 UTF-8。返回 1=合法, 0=非法 */
static int is_valid_utf8(const char *src, size_t len) {
const unsigned char *p = (const unsigned char *)src;
const unsigned char *end = p + len;
while (p < end) {
unsigned char b = *p;
if (b < 0x80) { p++; }
else if (b >= 0xC2 && b <= 0xDF) {
if (p + 1 >= end || p[1] < 0x80 || p[1] > 0xBF) return 0;
p += 2;
} else if (b >= 0xE0 && b <= 0xEF) {
if (p + 2 >= end || p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) return 0;
p += 3;
} else if (b >= 0xF0 && b <= 0xF4) {
if (p + 3 >= end || p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF || p[3] < 0x80 || p[3] > 0xBF) return 0;
p += 4;
} else {
return 0; /* C0-C1, 80-BF, F5-FF */
}
}
return 1;
}

/* 用 popen 调用 file -bi 检测编码，返回检测到的 from_enc（malloc'd）。
* iso-8859-* 一律视为 gb18030（兼容 GBK）。失败返回 NULL。 */
static char *detect_encoding(const char *content, size_t len) {
char tmppath[256];
snprintf(tmppath, sizeof(tmppath), "/tmp/iconv_detect_%d.tmp", (int)getpid());
FILE *f = fopen(tmppath, "wb");
if (!f) return NULL;
fwrite(content, 1, len, f);
fclose(f);

char cmd[512];
snprintf(cmd, sizeof(cmd), "file -bi '%s' 2>/dev/null", tmppath);
FILE *pp = popen(cmd, "r");
if (!pp) { unlink(tmppath); return NULL; }

char mime[256] = {0};
if (fgets(mime, sizeof(mime), pp)) {
/* 转小写 */
for (char *c = mime; *c; c++) { if (*c >= 'A' && *c <= 'Z') *c += 32; }
}
pclose(pp);
unlink(tmppath);

/* 检查 charset */
if (strstr(mime, "charset=utf-8") || strstr(mime, "charset=us-ascii")) {
return util_strdup("utf-8");
}
/* 匹配已知中文/日文编码 */
const char *encs[] = {"gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr", NULL};
for (int i = 0; encs[i]; i++) {
char needle[32];
snprintf(needle, sizeof(needle), "charset=%s", encs[i]);
if (strstr(mime, needle)) return util_strdup(encs[i]);
}
/* iso-8859-* → 视为 gb18030（file 对中文文件常误报 iso-8859-1） */
if (strstr(mime, "charset=iso-8859")) return util_strdup("gb18030");
return NULL;
}

char *util_iconv_if_needed(const char *content) {
if (!content) return util_sanitize_utf8("");
size_t len = strlen(content);

/* UTF-8 合法 → 直接 sanitize（idempotent） */
if (is_valid_utf8(content, len)) {
return util_sanitize_utf8(content);
}

/* 非 UTF-8 → 尝试 iconv 转码 */
char *from_enc = detect_encoding(content, len);
if (!from_enc) from_enc = util_strdup("gb18030"); /* 默认按 gb18030 尝试 */

/* 构建 iconv 管道：echo content | iconv -f gb18030 -t UTF-8//IGNORE */
char tmppath[256];
snprintf(tmppath, sizeof(tmppath), "/tmp/iconv_in_%d.tmp", (int)getpid());
FILE *f = fopen(tmppath, "wb");
if (!f) { free(from_enc); return util_sanitize_utf8(content); }
fwrite(content, 1, len, f);
fclose(f);

char cmd[512];
snprintf(cmd, sizeof(cmd), "iconv -f '%s' -t UTF-8//IGNORE '%s' 2>/dev/null", from_enc, tmppath);
free(from_enc);

FILE *pp = popen(cmd, "r");
if (!pp) { unlink(tmppath); return util_sanitize_utf8(content); }

StrBuf sb;
sb_init(&sb);
char buf2[4096];
size_t total = 0;
while (fgets(buf2, sizeof(buf2), pp)) {
sb_append(&sb, buf2);
total += strlen(buf2);
}
pclose(pp);
unlink(tmppath);

if (total == 0 && len > 0) {
/* iconv 失败 → 兜底 */
sb_free(&sb);
return util_sanitize_utf8(content);
}

char *result = util_sanitize_utf8(sb.data);
sb_free(&sb);
return result;
}

/* ============================================================
* 工具函数
* ============================================================ */
Expand Down
4 changes: 4 additions & 0 deletions c/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ void util_truncate_chars(char *s, int max_chars);
/* UTF-8 sanitize：将非法字节替换为 \ufffd 字面文本，返回新 malloc'd 字符串 */
char *util_sanitize_utf8(const char *src);

/* 编码检测+转换：若 content 非 UTF-8（如 GBK），调用系统 iconv 转码后 sanitize。
* 转码失败则 fallback 到 sanitize_utf8（保证 JSON 安全）。返回新 malloc'd 字符串。 */
char *util_iconv_if_needed(const char *content);

/* trim 尾部空白 */
char *util_rtrim(char *s);

Expand Down
55 changes: 54 additions & 1 deletion go/tools.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"strings"
"syscall"
"time"
"unicode/utf8"
)

const defaultToolResultMaxBytes = 100000
Expand Down Expand Up @@ -188,6 +189,7 @@ func (td *ToolDispatcher) CallSummary(name string, params map[string]string) str
// ─── 工具实现 ───

// toolRead 读取文件（支持 offset/limit）
// 非 UTF-8 编码（GBK 等）文件自动调用 iconv 转码，失败则 sanitize 兜底
func (td *ToolDispatcher) toolRead(p, offsetStr, limitStr string) (string, error) {
if p == "" {
return "", fmt.Errorf("no path provided")
Expand All @@ -201,7 +203,20 @@ func (td *ToolDispatcher) toolRead(p, offsetStr, limitStr string) (string, error
return "", fmt.Errorf("read error: %v", err)
}

lines := strings.Split(string(data), "\n")
// 编码处理：UTF-8 合法 → 直接用；否则尝试 iconv 转码，失败 fallback sanitize
var content string
if utf8.Valid(data) {
content = sanitizeUTF8(data)
} else {
converted, ok := iconvToUTF8(data)
if ok {
content = converted
} else {
content = sanitizeUTF8(data) // 兜底：保证 JSON 序列化不破坏
}
}

lines := strings.Split(content, "\n")
// offset 默认 1
offset := 1
if offsetStr != "" {
Expand Down Expand Up @@ -746,6 +761,44 @@ func (td *ToolDispatcher) toolSubAgent(ctx context.Context, prompt, description,
return "", fmt.Errorf("sub-agent launcher not configured")
}

// ─── iconvToUTF8: 调用系统 iconv 将非 UTF-8 编码（GBK/GB18030 等）转为 UTF-8 ───
// file 命令对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集。
// 返回 (sanitized 内容, 是否成功)。失败时 ok=false，调用方应 fallback 到 sanitizeUTF8。
func iconvToUTF8(data []byte) (string, bool) {
// 检测编码：优先用 file -bi，iso-8859-* 一律按 gb18030 处理（兼容 GBK）
fromEnc := "gb18030"
if tmpfile, err := os.CreateTemp("", "iconv-*.txt"); err == nil {
tmpfile.Write(data)
tmpfile.Close()
defer os.Remove(tmpfile.Name())
if out, err := exec.Command("file", "-bi", tmpfile.Name()).Output(); err == nil {
mime := strings.ToLower(string(out))
if strings.Contains(mime, "charset=utf-8") || strings.Contains(mime, "charset=us-ascii") {
return sanitizeUTF8(data), true // 实际是合法 UTF-8/ASCII
}
// 提取 charset，若明确非 iso-8859 则用检测到的值
for _, enc := range []string{"gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"} {
if strings.Contains(mime, "charset="+enc) {
fromEnc = enc
break
}
}
}
}
// 调用 iconv 转码
cmd := exec.Command("iconv", "-f", fromEnc, "-t", "UTF-8//IGNORE")
cmd.Stdin = bytes.NewReader(data)
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return "", false
}
if out.Len() == 0 && len(data) > 0 {
return "", false
}
return sanitizeUTF8(out.Bytes()), true
}

// ─── sanitizeUTF8: 过滤非法 UTF-8 字节，替换为字面 \ufffd ───
// 移植自 src/awk/sanitize_utf8.awk，确保 JSON 序列化不会因非法字节失败。
func sanitizeUTF8(data []byte) string {
Expand Down
60 changes: 57 additions & 3 deletions rust/src/tools.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,23 @@ use crate::config::Config;
if path.is_empty() {
bail!("no path provided");
}
let data = fs::read_to_string(path)
let data = fs::read(path)
.map_err(|_| anyhow!("Error: file not found or unreadable: {path}"))?;
// 编码处理：UTF-8 合法 → 直接用；否则尝试 iconv 转码，失败 fallback sanitize
let content = match std::str::from_utf8(&data) {
Ok(s) => self.sanitize_utf8(data.as_slice()),
Err(_) => {
match self.iconv_to_utf8(&data) {
Some(converted) => converted,
None => self.sanitize_utf8(data.as_slice()),
}
}
};
if offset.is_none() && limit.is_none() {
return Ok(data);
return Ok(content);
}

let mut lines: Vec<&str> = data.split('\n').collect();
let mut lines: Vec<&str> = content.split('\n').collect();
if !lines.is_empty() && lines.last().map(|l| l.is_empty()).unwrap_or(false) {
lines.pop();
}
Expand Down Expand Up @@ -1101,6 +1111,50 @@ use crate::config::Config;
out
}

/// iconv_to_utf8: 调用系统 iconv 将非 UTF-8 编码（GBK/GB18030 等）转为 UTF-8。
/// file 命令对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集。
/// 返回 Some(sanitized_content) 或 None（失败时调用方应 fallback 到 sanitize_utf8）。
fn iconv_to_utf8(&self, data: &[u8]) -> Option<String> {
// 检测编码：file -bi，iso-8859-* 一律按 gb18030 处理（兼容 GBK）
let mut from_enc = "gb18030";
// 写临时文件用 file 检测
let tmp = std::env::temp_dir().join(format!("iconv-{}.tmp", std::process::id()));
if fs::write(&tmp, data).is_ok() {
if let Ok(out) = Command::new("file").arg("-bi").arg(&tmp).output() {
let mime = String::from_utf8_lossy(&out.stdout).to_lowercase();
if mime.contains("charset=utf-8") || mime.contains("charset=us-ascii") {
let _ = fs::remove_file(&tmp);
return Some(self.sanitize_utf8(data));
}
for enc in &["gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"] {
if mime.contains(&format!("charset={enc}")) {
from_enc = enc;
break;
}
}
}
let _ = fs::remove_file(&tmp);
}
// 调用 iconv 转码
let child = Command::new("iconv")
.arg("-f").arg(from_enc)
.arg("-t").arg("UTF-8//IGNORE")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.ok()?;
use std::io::Write;
let mut stdin = child.stdin?;
stdin.write_all(data).ok()?;
drop(stdin);
let output = child.wait_with_output().ok()?;
if output.stdout.is_empty() && !data.is_empty() {
return None;
}
Some(self.sanitize_utf8(&output.stdout))
}

/// sanitize_utf8: replace illegal UTF-8 bytes with literal `\ufffd` text.
/// Ported from src/awk/sanitize_utf8.awk — ensures JSON serialization won't
/// fail on illegal bytes and produces the same `\ufffd` text the mock server checks for.
Expand Down
35 changes: 33 additions & 2 deletions src/agent.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -892,15 +892,46 @@ tool_bash_mode_allows() {
(( (8#$required & (4095 ^ 8#$allowed)) == 0 ))
}

util_iconv_if_needed() {
# 读 stdin，若检测到非 UTF-8 编码则 iconv 转码，再 sanitize。
# 常见中文编码: iso-8859-1/iso-8859-x 在 file 命令里可能报 iso-8859，但实际是 GBK/GB18030。
local tmp; tmp=$(mktemp)
cat > "$tmp"
local enc=""
if command -v file >/dev/null 2>&1; then
enc=$(file -bi "$tmp" 2>/dev/null | sed -n 's/.*charset=\([^[:space:];]*\).*/\1/p')
fi
enc="${enc:-unknown}"
enc=$(printf '%s' "$enc" | tr 'A-Z' 'a-z')
# 已是 UTF-8 / ascii / unknown → 直接走 sanitize（idempotent，UTF-8 不受影响）
if [[ "$enc" == "utf-8" || "$enc" == "us-ascii" || "$enc" == "unknown" ]]; then
util_awk_run -f "$AWK_DIR/sanitize_utf8.awk" "$tmp"
else
# 非 UTF-8 编码（gbk/gb18030/iso-8859-x 等）→ 先 iconv 转码
# file 对中文文件常报 iso-8859-1，实际多为 GBK/GB18030；GB18030 是 GBK 超集，优先用
local from_enc="$enc"
if [[ "$enc" == iso-8859* ]]; then
from_enc="gb18030" # GB18030 兼容 GBK 和绝大多数中文编码
fi
if iconv -f "$from_enc" -t UTF-8//IGNORE "$tmp" 2>/dev/null | util_awk_run -f "$AWK_DIR/sanitize_utf8.awk"; then
: # success
else
# iconv 失败 → 最后兜底：直接 sanitize 原始字节
util_awk_run -f "$AWK_DIR/sanitize_utf8.awk" "$tmp"
fi
fi
rm -f "$tmp"
}

tool_read() {
local path="$1" offset="${2:-1}" limit="${3:-0}"
[[ -z "$path" ]] && { echo "no path provided"; return 1; }
[[ ! -f "$path" ]] && { echo "file not found: $path"; return 1; }
[[ ! -r "$path" ]] && { echo "permission denied: $path"; return 1; }
if (( limit > 0 )); then
sed -n "${offset},$((offset + limit - 1))p" "$path"
sed -n "${offset},$((offset + limit - 1))p" "$path" | util_iconv_if_needed
else
sed -n "${offset},\$p" "$path"
sed -n "${offset},\$p" "$path" | util_iconv_if_needed
fi
}

Expand Down
Loading
Loading