diff --git a/.github/workflows/check-code.yml b/.github/workflows/check-code.yml index 1841e2b..2d1b060 100644 --- a/.github/workflows/check-code.yml +++ b/.github/workflows/check-code.yml @@ -5,7 +5,7 @@ on: jobs: check-i18n: - name: Check Translation Files + name: Check i18n translations runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -15,3 +15,37 @@ jobs: python-version: "3.x" - run: python3 tests/check_i18n.py + + check-json: + name: Validate JSON files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - run: python3 tests/check_json.py + + check-trailing-newline: + name: Check trailing newlines + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - run: python3 tests/check_trailing_newline.py + + biome-lint: + name: Biome lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: biomejs/setup-biome@v2 + + - run: biome lint src/ diff --git a/.gitignore b/.gitignore index c47fa43..56c7984 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __MACOSX/ .DS_Store desktop.ini Thumbs.db + +tests/.biome/ diff --git a/README.md b/README.md index 72fc742..c5e1233 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,78 @@ -A Pseudo-localization Translater Demo +# Pseudo Localization Demo 一个伪本地化演示 -伪本地化（pseudo-localization，语言环境名称为 qps-ploc, qps-plocm, qps-ploca, en-XA, en-XB），是模拟本地化过程的一种方式。而通过模拟本地化过程，能够有效地调查在本地化中出现的问题（如字符无法正常显示，或因字符串过长而导致语段显示不完整等）。 +伪本地化 (pseudo-localization, 语言环境名称为 qps-ploc, qps-plocm, qps-ploca, en-XA, en-XB), 是模拟本地化过程的一种方式。而通过模拟本地化过程, 能够有效地调查在本地化中出现的问题 (如字符无法正常显示, 或因字符串过长而导致语段显示不完整等)。 -在伪本地化过程中，英文字母会被替换为来自其他语言的重音符号和字符。（例如，字母 a 可以被 αäáàāāǎǎăăåå 中的任何一个替换。），还会添加分隔符等以增加字符串长度。 -举例：“Windows 照片库（Windows Photo Gallery）”→“ [1iaT9][ Ẅĭпðøωś Þнôтŏ Ģάŀļєяÿ !!! !] ” +在伪本地化过程中, 英文字母会被替换为来自其他语言的重音符号和字符 (例如, 字母 a 可以被 αäáàāāǎǎăăåå 中的任何一个替换), 还会添加分隔符等以增加字符串长度。 +举例: "Windows 照片库 (Windows Photo Gallery)"→" [1iaT9][ Ẅĭпðøωś Þнôтŏ Ģάŀļєяÿ !!! !] " -该网页演示了伪本地化的一部分，即用不同的字符替换英文字母和添加分隔符。 +该网页演示了伪本地化的一部分, 即用不同的字符替换英文字母和添加分隔符。 -更多功能将在之后更新，感谢大家的支持！ +此工具不会上传你的任何数据。 + +## 使用 + +如果想要在线预览, 请访问: https://suntrise.github.io/pseudo/ + +如果需要在本地使用, 且您安装了 Python 3, 可以直接执行: +~~~bash +python3 -m http.server 8000 +~~~ +在 `localhost:8000` 中预览页面 + +## 开发 + +### 环境准备 + +| 工具 | 用途 | 安装方式 | +|------|------|----------| +| Python 3 | 运行检查脚本, 或启用本地服务器 | [python.org](https://www.python.org/) | + +### 进行本地化 + +翻译文件位于 `data/i18n.json`, 您可以简单的编辑此 JSON 文件进行进行开发。 + +编辑翻译后, 请运行 i18n 检查以确保翻译质量: + +~~~bash +python3 tests/check_i18n.py +~~~ + +检查内容包括: +- 全角字符检测 +- 缺失/多余的翻译键 +- 空值检测 +- 占位符一致性 +- 首尾空白字符 + +### 开发功能 + +开发完成后, 请执行以下检查确保代码质量: + +~~~bash +# i18n 翻译检查 +python3 tests/check_i18n.py + +# JSON 文件检查 (语法 + 重复键) +python3 tests/check_json.py + +# 文件尾换行符检查 +python3 tests/check_trailing_newline.py + +# JavaScript 代码检查 (首次运行会自动下载 Biome) +python3 tests/check_biome.py +~~~ + +> **Windows 用户** 可能需要将 `python3` 替换为 `python` + +### 检查说明 + +| 检查项 | 脚本 | 说明 | +|--------|------|------| +| i18n 翻译 | `tests/check_i18n.py` | 检查翻译完整性、占位符一致性、全角字符等 | +| JSON 验证 | `tests/check_json.py` | 检查 JSON 语法和重复键 | +| 换行符 | `tests/check_trailing_newline.py` | 检查文本文件是否以换行符结尾 | +| JS 代码 | `tests/check_biome.py` | 检查 JavaScript 代码质量 (自动下载 Biome)| + +所有检查会在 Pull Request 时自动运行。 diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..e471b21 --- /dev/null +++ b/biome.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.2/schema.json", + "files": { + "includes": ["src/**/*.js"] + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": { + "useTemplate": "warn", + "useConst": "warn" + }, + "complexity": { + "useLiteralKeys": "warn" + }, + "correctness": { + "noUnusedImports": "warn", + "noUnusedVariables": "warn", + "useParseIntRadix": "warn" + }, + "suspicious": { + "useIterableCallbackReturn": "warn" + } + } + }, + "formatter": { + "enabled": false + } +} diff --git a/index.html b/index.html index ae2bbbc..fc938e0 100644 --- a/index.html +++ b/index.html @@ -218,4 +218,4 @@

- \ No newline at end of file + diff --git a/tests/check_biome.py b/tests/check_biome.py new file mode 100644 index 0000000..830c34c --- /dev/null +++ b/tests/check_biome.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +import platform +import subprocess +import sys +import os +import ssl +import urllib.request +import stat +import shutil +import time +from pathlib import Path + +BIOME_VERSION = "2.4.2" +BIOME_DIR = Path(__file__).parent / ".biome" +TIMEOUT = 30 +MAX_RETRIES = 3 + +def get_platform(): + system, machine = platform.system().lower(), platform.machine().lower() + platform_map = {"darwin": "darwin", "linux": "linux", "windows": "win32"} + platform_name = platform_map.get(system) + if not platform_name: + sys.exit(f"Unsupported OS: {system}") + if platform_name == "linux": + try: + if Path("/proc/self/maps").exists(): + with open("/proc/self/maps") as f: + if "musl" in f.read(): + platform_name = "linux-musl" + except Exception: + pass + arch_map = {"x86_64": "x64", "amd64": "x64", "arm64": "arm64", "aarch64": "arm64"} + arch = arch_map.get(machine) + if not arch: + sys.exit(f"Unsupported architecture: {machine}") + return platform_name, arch + +def find_biome(): + if path := shutil.which("biome"): + return Path(path) + candidate = Path.home() / ".biome" / "bin" / "biome" + if candidate.exists() and os.access(candidate, os.X_OK): + return candidate + return None + +def check_version(executable, version): + try: + result = subprocess.run( + [str(executable), "--version"], + capture_output=True, text=True, timeout=10, check=True + ) + return version in result.stdout + except Exception: + return False + +def download_file(url, target): + target.parent.mkdir(parents=True, exist_ok=True) + for attempt in range(MAX_RETRIES): + try: + ctx = ssl.create_default_context() + with urllib.request.urlopen(url, context=ctx, timeout=TIMEOUT) as resp: + tmp_path = target.with_suffix(target.suffix + ".tmp") + with open(tmp_path, "wb") as f: + f.write(resp.read()) + tmp_path.replace(target) + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + except Exception: + if attempt < MAX_RETRIES - 1: + time.sleep(2 ** attempt) + continue + return False + +def download_biome(): + platform_name, arch = get_platform() + ext = ".exe" if platform.system().lower() == "windows" else "" + suffix = "-musl" if platform_name == "linux-musl" else "" + filename = f"biome-{platform_name}-{arch}{suffix}{ext}" + url = f"https://github.com/biomejs/biome/releases/download/biome@{BIOME_VERSION}/{filename}" + target = BIOME_DIR / f"biome{ext}" + if target.exists() and check_version(target, BIOME_VERSION): + return target + if not download_file(url, target): + sys.exit(f"Failed to download: {url}") + if platform.system().lower() != "windows": + target.chmod(target.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return target + +def install_via_package_manager(): + system = platform.system().lower() + if system == "darwin" and shutil.which("brew"): + subprocess.run(["brew", "install", "biome"], capture_output=True, timeout=180) + return True + if system == "windows": + for manager, cmd in [("choco", ["choco", "install", "biome", "-y"]), ("scoop", ["scoop", "install", "biome"])]: + if shutil.which(manager): + subprocess.run(cmd, capture_output=True, timeout=180) + return True + if shutil.which("curl"): + subprocess.run( + "sh -c 'curl -fsSL https://biomejs.dev/install.sh | sh'", + shell=True, capture_output=True, timeout=120 + ) + return True + return False + +def run_lint(executable): + src_dir = Path(__file__).parent.parent / "src" + if not src_dir.exists(): + return 0 + result = subprocess.run([str(executable), "lint", str(src_dir)]) + return result.returncode + +def main(): + if exe := find_biome(): + if check_version(exe, BIOME_VERSION): + sys.exit(run_lint(exe)) + install_via_package_manager() + if exe := find_biome(): + if check_version(exe, BIOME_VERSION): + sys.exit(run_lint(exe)) + exe = download_biome() + sys.exit(run_lint(exe)) + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130) + except Exception as e: + sys.exit(2) diff --git a/tests/check_i18n.py b/tests/check_i18n.py index 78c336d..05ec3fd 100755 --- a/tests/check_i18n.py +++ b/tests/check_i18n.py @@ -7,26 +7,14 @@ I18N_FILE = "data/i18n.json" FULLWIDTH_RANGES = [ - (0xFF01, 0xFF60), + (0xFF01, 0xFF5F), (0xFFE0, 0xFFE6), ] -FULLWIDTH_WHITELIST = set([ - "\u3002", - "\u300C", - "\u300D", - "\u300A", - "\u300B", - "\u3001", - "\u00B7", -]) - PLACEHOLDER_PATTERN = re.compile(r"\{[^}]+\}") def is_fullwidth_char(ch): - if ch in FULLWIDTH_WHITELIST: - return False cp = ord(ch) for start, end in FULLWIDTH_RANGES: if start <= cp <= end: @@ -38,12 +26,15 @@ def check_fullwidth(locales): warnings = [] for lang, entries in locales.items(): for key, value in entries.items(): + normalized_value = None for i, ch in enumerate(value): if is_fullwidth_char(ch): + if normalized_value is None: + normalized_value = unicodedata.normalize('NFKC', value) warnings.append( - f"[{lang}.{key}] pos {i}: '{ch}' " - f"(U+{ord(ch):04X} {unicodedata.name(ch, 'UNKNOWN')})" + f"[{lang}.{key}] pos {i}: '{ch}' (U+{ord(ch):04X} {unicodedata.name(ch, 'UNKNOWN')})" f"\n context: ...{value[max(0, i - 10):i + 11]}..." + f"\n Suggest: Replace entire string with: {repr(normalized_value)}" ) return warnings @@ -56,9 +47,15 @@ def check_missing_keys(locales, base_lang="en"): continue current_keys = set(entries.keys()) for key in sorted(base_keys - current_keys): - warnings.append(f"[{lang}] missing key: '{key}'") + warnings.append( + f"[{lang}] missing key: '{key}'" + f"\n Suggest: Add translation for '{key}' from {base_lang}" + ) for key in sorted(current_keys - base_keys): - warnings.append(f"[{lang}] extra key: '{key}' (not in {base_lang})") + warnings.append( + f"[{lang}] extra key: '{key}' (not in {base_lang})" + f"\n Suggest: Remove key or add to {base_lang} source" + ) return warnings @@ -67,7 +64,10 @@ def check_empty_values(locales): for lang, entries in locales.items(): for key, value in entries.items(): if not value or not value.strip(): - warnings.append(f"[{lang}.{key}] empty or whitespace-only value") + warnings.append( + f"[{lang}.{key}] empty or whitespace-only value" + "\n Suggest: Provide valid translated content" + ) return warnings @@ -83,9 +83,15 @@ def check_placeholders(locales, base_lang="en"): base_ph = set(PLACEHOLDER_PATTERN.findall(base[key])) curr_ph = set(PLACEHOLDER_PATTERN.findall(value)) for ph in sorted(base_ph - curr_ph): - warnings.append(f"[{lang}.{key}] missing placeholder: {ph}") + warnings.append( + f"[{lang}.{key}] missing placeholder: {ph}" + f"\n Suggest: Insert {ph} at appropriate position (match {base_lang} structure)" + ) for ph in sorted(curr_ph - base_ph): - warnings.append(f"[{lang}.{key}] extra placeholder: {ph}") + warnings.append( + f"[{lang}.{key}] extra placeholder: {ph}" + f"\n Suggest: Remove {ph} to align with {base_lang} source" + ) return warnings @@ -94,7 +100,6 @@ def check_whitespace(locales): for lang, entries in locales.items(): for key, value in entries.items(): if value != value.strip(): - stripped = value.strip() leading = value[:len(value) - len(value.lstrip())] trailing = value[len(value.rstrip()):] detail = [] @@ -102,11 +107,13 @@ def check_whitespace(locales): detail.append(f"leading {repr(leading)}") if trailing: detail.append(f"trailing {repr(trailing)}") - warnings.append(f"[{lang}.{key}] {', '.join(detail)}") + warnings.append( + f"[{lang}.{key}] {', '.join(detail)}" + "\n Suggest: Trim unnecessary whitespace characters" + ) return warnings - def main(): try: with open(I18N_FILE, "r", encoding="utf-8") as f: diff --git a/tests/check_json.py b/tests/check_json.py new file mode 100644 index 0000000..63e71de --- /dev/null +++ b/tests/check_json.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import json +import sys +import glob + +DATA_DIR = "data" + +GREEN = "\033[32m" +YELLOW = "\033[33m" +RESET = "\033[0m" + + +def find_duplicate_keys(content): + duplicates = [] + + def check_pairs(pairs): + seen = {} + obj = {} + for key, value in pairs: + if key in seen: + duplicates.append((key, seen[key] + 1)) + seen[key] = seen.get(key, 0) + 1 + obj[key] = value + return obj + + json.loads(content, object_pairs_hook=check_pairs) + return duplicates + + +def main(): + files = sorted(glob.glob(f"{DATA_DIR}/**/*.json", recursive=True)) + if not files: + print(f"WARNING: No JSON files found in {DATA_DIR}/") + sys.exit(2) + + has_errors = False + for path in files: + warnings = [] + try: + with open(path, "r", encoding="utf-8") as f: + content = f.read() + dupes = find_duplicate_keys(content) + for key, count in dupes: + warnings.append(f"duplicate key: '{key}' (appeared {count + 1} times)") + except json.JSONDecodeError as e: + warnings.append(f"invalid JSON: {e}") + + if warnings: + has_errors = True + print(f"{YELLOW}[WARN]{RESET} {path}:") + for w in warnings: + print(f" {w}") + else: + print(f"{GREEN}[PASS]{RESET} {path}") + + if has_errors: + sys.exit(1) + else: + print(f"\nAll JSON files are valid.") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/check_trailing_newline.py b/tests/check_trailing_newline.py new file mode 100644 index 0000000..6897d45 --- /dev/null +++ b/tests/check_trailing_newline.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import sys +import glob + +TEXT_EXTENSIONS = { + ".html", ".css", ".js", ".json", ".md", ".yml", ".yaml", ".txt", ".py", +} + +SCAN_PATTERNS = [ + "*.html", + "*.css", + "*.md", + "*.yml", + "*.yaml", + "*.txt", + "src/**/*.js", + "data/**/*.json", + "tests/**/*.py", + ".github/**/*.yml", +] + +GREEN = "\033[32m" +YELLOW = "\033[33m" +RESET = "\033[0m" + + +def main(): + files = set() + for pattern in SCAN_PATTERNS: + files.update(glob.glob(pattern, recursive=True)) + + if not files: + print("WARNING: No text files found.") + sys.exit(2) + + warnings = [] + for path in sorted(files): + try: + with open(path, "rb") as f: + content = f.read() + except OSError: + continue + + if not content: + continue + + if not content.endswith(b"\n"): + warnings.append(path) + + if warnings: + print(f"{YELLOW}[WARN]{RESET} Missing trailing newline ({len(warnings)}):\n") + for path in warnings: + print(f" {path}") + print() + sys.exit(1) + else: + print(f"{GREEN}[PASS]{RESET} All text files end with a newline") + print(f"\nAll files are valid.") + sys.exit(0) + + +if __name__ == "__main__": + main()