diff --git a/.changeset/i18n-docs-support.md b/.changeset/i18n-docs-support.md new file mode 100644 index 0000000..23b8cfd --- /dev/null +++ b/.changeset/i18n-docs-support.md @@ -0,0 +1,5 @@ +--- +"leadtype": patch +--- + +Add first-class docs i18n support with locale-aware generation, localized source loading, per-locale search/LLM/readability artifacts, and a new `leadtype/i18n` helper surface. Locale-scoped search generation now uses URL-path document ids to align generated indexes with the source API. diff --git a/bun.lock b/bun.lock index 8488e80..b3dfb55 100644 --- a/bun.lock +++ b/bun.lock @@ -10,6 +10,7 @@ "@changesets/cli": "2.31.0", "@mdx-js/react": "^3.1.1", "@typescript/native-preview": "7.0.0-dev.20260509.2", + "@vitest/coverage-v8": "4.1.5", "husky": "^9.1.7", "leadtype": "workspace:*", "turbo": "^2.9.12", @@ -132,7 +133,7 @@ }, "packages/leadtype": { "name": "leadtype", - "version": "0.1.1", + "version": "0.1.2", "bin": "./dist/cli.js", "dependencies": { "@types/mdast": "4.0.4", @@ -235,7 +236,7 @@ "@babel/helpers": ["@babel/helpers@7.29.2", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.29.0" } }, "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw=="], - "@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/parser": ["@babel/parser@7.29.3", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA=="], "@babel/plugin-syntax-jsx": ["@babel/plugin-syntax-jsx@7.28.6", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w=="], @@ -249,6 +250,8 @@ "@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], + "@bcoe/v8-coverage": ["@bcoe/v8-coverage@1.0.2", "", {}, "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA=="], + "@biomejs/biome": ["@biomejs/biome@2.4.14", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.14", "@biomejs/cli-darwin-x64": "2.4.14", "@biomejs/cli-linux-arm64": "2.4.14", "@biomejs/cli-linux-arm64-musl": "2.4.14", "@biomejs/cli-linux-x64": "2.4.14", "@biomejs/cli-linux-x64-musl": "2.4.14", "@biomejs/cli-win32-arm64": "2.4.14", "@biomejs/cli-win32-x64": "2.4.14" }, "bin": { "biome": "bin/biome" } }, "sha512-TmAvxOEgrpLypzVGJ8FulIZnlyA9TxrO1hyqYrCz9r+bwma9xXxuLA5IuYnj55XQneFx460KjRbx6SWGLkg3bQ=="], "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.14", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XvgoE9XOawUOQPdmvs4J7wPhi/DLwSCGks3AlPJDmh34O0awRTqCED1HRcRDdpf1Zrp4us4MGOOdIxNpbqNF5Q=="], @@ -961,6 +964,8 @@ "@vitejs/plugin-react": ["@vitejs/plugin-react@6.0.1", "", { "dependencies": { "@rolldown/pluginutils": "1.0.0-rc.7" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler"] }, "sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ=="], + "@vitest/coverage-v8": ["@vitest/coverage-v8@4.1.5", "", { "dependencies": { "@bcoe/v8-coverage": "^1.0.2", "@vitest/utils": "4.1.5", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", "istanbul-reports": "^3.2.0", "magicast": "^0.5.2", "obug": "^2.1.1", "std-env": "^4.0.0-rc.1", "tinyrainbow": "^3.1.0" }, "peerDependencies": { "@vitest/browser": "4.1.5", "vitest": "4.1.5" }, "optionalPeers": ["@vitest/browser"] }, "sha512-38C0/Ddb7HcRG0Z4/DUem8x57d2p9jYgp18mkaYswEOQBGsI1CG4f/hjm0ZCeaJfWhSZ4k7jgs29V1Zom7Ki9A=="], + "@vitest/expect": ["@vitest/expect@4.1.5", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.5", "@vitest/utils": "4.1.5", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw=="], "@vitest/mocker": ["@vitest/mocker@4.1.5", "", { "dependencies": { "@vitest/spy": "4.1.5", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw=="], @@ -999,6 +1004,8 @@ "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], + "ast-v8-to-istanbul": ["ast-v8-to-istanbul@1.0.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.31", "estree-walker": "^3.0.3", "js-tokens": "^10.0.0" } }, "sha512-1fSfIwuDICFA4LKkCzRPO7F0hzFf0B7+Xqrl27ynQaa+Rh0e1Es0v6kWHPott3lU10AyAr7oKHa65OppjLn3Rg=="], + "astring": ["astring@1.9.0", "", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], "babel-dead-code-elimination": ["babel-dead-code-elimination@1.0.12", "", { "dependencies": { "@babel/core": "^7.23.7", "@babel/parser": "^7.23.6", "@babel/traverse": "^7.23.7", "@babel/types": "^7.23.6" } }, "sha512-GERT7L2TiYcYDtYk1IpD+ASAYXjKbLTDPhBtYj7X1NuRMDTMtAx9kyBenub1Ev41lo91OHCKdmP+egTDmfQ7Ig=="], @@ -1355,6 +1362,8 @@ "hachure-fill": ["hachure-fill@0.5.2", "", {}, "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg=="], + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + "hast-util-from-parse5": ["hast-util-from-parse5@8.0.3", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "devlop": "^1.0.0", "hastscript": "^9.0.0", "property-information": "^7.0.0", "vfile": "^6.0.0", "vfile-location": "^5.0.0", "web-namespaces": "^2.0.0" } }, "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg=="], "hast-util-parse-selector": ["hast-util-parse-selector@4.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A=="], @@ -1379,6 +1388,8 @@ "hookable": ["hookable@6.1.1", "", {}, "sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ=="], + "html-escaper": ["html-escaper@2.0.2", "", {}, "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="], + "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], "html-void-elements": ["html-void-elements@3.0.0", "", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], @@ -1435,9 +1446,15 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "istanbul-lib-coverage": ["istanbul-lib-coverage@3.2.2", "", {}, "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg=="], + + "istanbul-lib-report": ["istanbul-lib-report@3.0.1", "", { "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" } }, "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw=="], + + "istanbul-reports": ["istanbul-reports@3.2.0", "", { "dependencies": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" } }, "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA=="], + "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "js-tokens": ["js-tokens@10.0.0", "", {}, "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q=="], "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], @@ -1513,6 +1530,10 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "magicast": ["magicast@0.5.3", "", { "dependencies": { "@babel/parser": "^7.29.3", "@babel/types": "^7.29.0", "source-map-js": "^1.2.1" } }, "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw=="], + + "make-dir": ["make-dir@4.0.0", "", { "dependencies": { "semver": "^7.5.3" } }, "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw=="], + "markdown-extensions": ["markdown-extensions@2.0.0", "", {}, "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q=="], "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], @@ -1957,6 +1978,8 @@ "stylis": ["stylis@4.4.0", "", {}, "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA=="], + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "tagged-tag": ["tagged-tag@1.0.0", "", {}, "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng=="], "tailwind-merge": ["tailwind-merge@3.5.0", "", {}, "sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A=="], @@ -2123,12 +2146,22 @@ "@antfu/install-pkg/package-manager-detector": ["package-manager-detector@1.6.0", "", {}, "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA=="], + "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + + "@babel/core/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@babel/generator/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/helper-compilation-targets/lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@babel/template/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + + "@babel/traverse/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="], "@manypkg/find-root/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="], @@ -2213,12 +2246,16 @@ "@tanstack/router-plugin/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], + "@tanstack/router-utils/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@tanstack/start-plugin-core/@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], "@tanstack/start-plugin-core/@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.40", "", {}, "sha512-s3GeJKSQOwBlzdUrj4ISjJj5SfSh+aqn0wjOar4Bx95iV1ETI7F6S/5hLcfAxZ9kXDcyrAkxPlqmd1ZITttf+w=="], "anymatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], + "babel-dead-code-elimination/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "bash-tool/just-bash": ["just-bash@2.14.2", "", { "dependencies": { "diff": "^8.0.2", "fast-xml-parser": "^5.3.3", "file-type": "^21.2.0", "ini": "^6.0.0", "minimatch": "^10.1.1", "modern-tar": "^0.7.3", "papaparse": "^5.5.3", "quickjs-emscripten": "^0.32.0", "re2js": "^1.2.1", "seek-bzip": "^2.0.0", "smol-toml": "^1.6.0", "sprintf-js": "^1.1.3", "sql.js": "^1.13.0", "turndown": "^7.2.2", "yaml": "^2.8.2" }, "optionalDependencies": { "@mongodb-js/zstd": "^7.0.0", "node-liblzma": "^2.0.3" }, "bin": { "just-bash": "dist/bin/just-bash.js", "just-bash-shell": "dist/bin/shell/shell.js" } }, "sha512-9Na1rH03Ta5ydHTNotJ7dms1iZwb2kToOnKbnS29AlrCvi1CQ21Fm2lfu4S4rfwDGHYi4E4evgTDC/DcDx8tuQ=="], "bash-tool/yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], @@ -2313,6 +2350,8 @@ "@tanstack/router-plugin/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], + "@tanstack/start-plugin-core/@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "bash-tool/just-bash/fast-xml-parser": ["fast-xml-parser@5.7.2", "", { "dependencies": { "@nodable/entities": "^2.1.0", "fast-xml-builder": "^1.1.5", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w=="], "cytoscape-fcose/cose-base/layout-base": ["layout-base@2.0.1", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="], diff --git a/package.json b/package.json index 1135ebc..15d5612 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,11 @@ "@biomejs/biome": "2.4.14", "@changesets/changelog-github": "0.7.0", "@changesets/cli": "2.31.0", - "leadtype": "workspace:*", "@mdx-js/react": "^3.1.1", "@typescript/native-preview": "7.0.0-dev.20260509.2", + "@vitest/coverage-v8": "4.1.5", "husky": "^9.1.7", + "leadtype": "workspace:*", "turbo": "^2.9.12", "typescript": "6.0.3", "ultracite": "7.6.5" diff --git a/packages/leadtype/package.json b/packages/leadtype/package.json index 58c42b0..11244dd 100644 --- a/packages/leadtype/package.json +++ b/packages/leadtype/package.json @@ -33,6 +33,10 @@ "types": "./dist/fumadocs/index.d.ts", "import": "./dist/fumadocs/index.js" }, + "./i18n": { + "types": "./dist/i18n/index.d.ts", + "import": "./dist/i18n/index.js" + }, "./remark": { "types": "./dist/remark/index.d.ts", "import": "./dist/remark/index.js" diff --git a/packages/leadtype/rollup.config.ts b/packages/leadtype/rollup.config.ts index 75b3873..18cc111 100644 --- a/packages/leadtype/rollup.config.ts +++ b/packages/leadtype/rollup.config.ts @@ -7,6 +7,7 @@ const entries = { index: "src/index.ts", "mdx/index": "src/mdx/index.ts", "fumadocs/index": "src/fumadocs/index.ts", + "i18n/index": "src/i18n/index.ts", "remark/index": "src/remark/index.ts", "convert/index": "src/convert/index.ts", "llm/index": "src/llm/index.ts", diff --git a/packages/leadtype/src/cli.test.ts b/packages/leadtype/src/cli.test.ts index 037b40b..87624f0 100644 --- a/packages/leadtype/src/cli.test.ts +++ b/packages/leadtype/src/cli.test.ts @@ -327,6 +327,97 @@ describe("leadtype CLI", () => { ); }); + it("generates locale-scoped i18n artifacts while keeping default URLs stable", async () => { + const srcDir = await createTempDir(); + const outDir = await createTempDir(); + const capture = createCapture(); + + await mkdir(path.join(srcDir, "docs"), { recursive: true }); + await writeFile( + path.join(srcDir, "docs", "docs.config.ts"), + `export default { + product: { + name: "Localized Product", + summary: "Localized product summary.", + }, + groups: [{ slug: "get-started", title: "Get Started" }], + i18n: { + defaultLocale: "en", + locales: ["en", "zh"], + }, +};` + ); + await writeMdxPage( + srcDir, + "quickstart.mdx", + 'title: "Quickstart"\ndescription: "English quickstart."\ngroup: get-started', + "English body." + ); + await writeMdxPage( + srcDir, + "setup.mdx", + 'title: "Setup"\ndescription: "English setup."\ngroup: get-started', + "English setup." + ); + await writeMdxPage( + srcDir, + "zh/quickstart.mdx", + 'title: "快速开始"\ndescription: "中文快速开始。"\ngroup: get-started', + "中文正文。" + ); + + const code = await runCli( + ["generate", "--src", srcDir, "--out", outDir, "--format", "json"], + capture.io + ); + + expect(code).toBe(0); + const result = JSON.parse(capture.stdout) as { + files: { i18nManifest?: string }; + }; + expect(result.files.i18nManifest).toBe( + path.join(outDir, "docs", "i18n-manifest.json") + ); + + const manifest = JSON.parse( + await readFile(path.join(outDir, "docs", "i18n-manifest.json"), "utf8") + ) as { + defaultLocale: string; + artifacts: Array<{ locale: string; searchIndex: string }>; + }; + expect(manifest.defaultLocale).toBe("en"); + expect(manifest.artifacts).toContainEqual( + expect.objectContaining({ + locale: "zh", + searchIndex: "/docs/zh/search-index.json", + }) + ); + + const defaultSummary = await readFile( + path.join(outDir, "docs", "llms.txt"), + "utf8" + ); + expect(defaultSummary).toContain("](/docs/quickstart.md)"); + + const zhSummary = await readFile( + path.join(outDir, "docs", "zh", "llms.txt"), + "utf8" + ); + expect(zhSummary).toContain("快速开始"); + expect(zhSummary).toContain("](/docs/zh/quickstart.md)"); + expect(zhSummary).not.toContain("Setup"); + + const zhSearch = JSON.parse( + await readFile( + path.join(outDir, "docs", "zh", "search-index.json"), + "utf8" + ) + ) as { documents: [string, string, string, string][] }; + expect(zhSearch.documents.map((entry) => entry[3])).toEqual([ + "/docs/zh/quickstart", + ]); + }); + it("lets --name and --summary override docs config product fields", async () => { const srcDir = await createTempDir(); const outDir = await createTempDir(); diff --git a/packages/leadtype/src/cli/generate.ts b/packages/leadtype/src/cli/generate.ts index 734c955..2bd6477 100644 --- a/packages/leadtype/src/cli/generate.ts +++ b/packages/leadtype/src/cli/generate.ts @@ -1,11 +1,12 @@ import { existsSync } from "node:fs"; -import { cp, mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { glob as fg } from "tinyglobby"; import type { Pluggable, PluggableList } from "unified"; import { convertAllMdx } from "../convert"; +import { type DocsI18nManifest, normalizeDocsI18nConfig } from "../i18n"; import { type DocsPathMount, normalizeDocsPath, @@ -91,6 +92,7 @@ type GenerateResult = { docsRobotsTxt?: string; docsSitemapMd?: string; docsSitemapXml?: string; + i18nManifest?: string; docsLlmsTxt?: string; llmsFullTxt?: string; llmsTxt?: string; @@ -141,6 +143,7 @@ type LoadedDocsConfig = { type ResolvedGenerateMetadata = { configPath?: string; groups: DocsGroup[]; + i18n?: DocsConfig["i18n"]; product: ProductInfo; typeTableBasePath?: string; typeTableStrict?: boolean; @@ -358,6 +361,9 @@ function validateDocsConfig(value: unknown, configPath: string): DocsConfig { } return { groups, + ...(value.i18n === undefined + ? {} + : { i18n: value.i18n as DocsConfig["i18n"] }), product, typeTableBasePath: typeof value.typeTableBasePath === "string" @@ -482,6 +488,7 @@ async function resolveGenerateMetadata( return { configPath: loadedConfig.path, groups: loadedConfig.config.groups, + i18n: loadedConfig.config.i18n, product: applyProductOverrides(loadedConfig.config.product, args), typeTableBasePath: loadedConfig.config.typeTableBasePath ? path.resolve(srcDir, loadedConfig.config.typeTableBasePath) @@ -707,6 +714,98 @@ async function copyMountedMarkdownMirrors( ); } +async function hasMarkdownFiles(dir: string): Promise { + if (!existsSync(dir)) { + return false; + } + const files = await fg("**/*.md", { + absolute: false, + cwd: dir, + onlyFiles: true, + }); + return files.length > 0; +} + +async function copyDefaultLocaleMarkdownAliases( + outDir: string, + defaultLocale: string +): Promise { + const docsDir = path.join(outDir, DEFAULT_DOCS_DIR); + const defaultLocaleDir = path.join(docsDir, defaultLocale); + if (!(await hasMarkdownFiles(defaultLocaleDir))) { + return; + } + + const rootMarkdownFiles = await fg("*.md", { + absolute: false, + cwd: docsDir, + onlyFiles: true, + }); + if (rootMarkdownFiles.length > 0) { + throw new Error( + `Ambiguous i18n default-locale output. Use either root docs files or docs/${defaultLocale}/ files for the default locale, not both.` + ); + } + + const files = await fg("**/*.md", { + absolute: false, + cwd: defaultLocaleDir, + onlyFiles: true, + }); + await Promise.all( + files.map(async (file) => { + const sourcePath = path.join(defaultLocaleDir, file); + const targetPath = path.join(docsDir, file); + await mkdir(path.dirname(targetPath), { recursive: true }); + await cp(sourcePath, targetPath); + }) + ); + await rm(defaultLocaleDir, { force: true, recursive: true }); +} + +function buildI18nManifest( + config: DocsConfig["i18n"] +): DocsI18nManifest | undefined { + const i18n = normalizeDocsI18nConfig(config); + if (!i18n) { + return; + } + return { + version: 1, + defaultLocale: i18n.defaultLocale, + locales: i18n.locales, + artifacts: i18n.locales.map((locale) => { + const isDefault = locale.code === i18n.defaultLocale; + const prefix = isDefault ? "/docs" : `/docs/${locale.code}`; + return { + locale: locale.code, + urlPrefix: prefix, + llmsTxt: `${prefix}/llms.txt`, + llmsFullTxt: isDefault ? "/llms-full.txt" : `${prefix}/llms-full.txt`, + searchIndex: `${prefix}/search-index.json`, + searchContent: `${prefix}/search-content.json`, + agentReadabilityManifest: `${prefix}/agent-readability.json`, + robotsTxt: `${prefix}/robots.txt`, + sitemapMd: `${prefix}/sitemap.md`, + sitemapXml: `${prefix}/sitemap.xml`, + }; + }), + }; +} + +async function writeI18nManifest( + outDir: string, + manifest: DocsI18nManifest | undefined +): Promise { + if (!manifest) { + return; + } + const outputPath = path.join(outDir, DEFAULT_DOCS_DIR, "i18n-manifest.json"); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(manifest, null, 2)}\n`); + return outputPath; +} + async function createSourceMirror( srcDir: string, sources: ResolvedDocsSource[], @@ -832,17 +931,26 @@ export async function runGenerateCommand( ...metadata, groups: await inferGroups(sourceMirror.docsDir), }; - - const navigation = await resolveDocsNavigation({ - srcDir: sourceMirror.srcDir, - groups, - mounts, - }); - const firstUnknownGroup = navigation.unknown[0]; - if (firstUnknownGroup) { - throw new Error( - `${firstUnknownGroup.urlPath} declares unknown group "${firstUnknownGroup.slug}"` - ); + const i18n = normalizeDocsI18nConfig(metadata.i18n); + const i18nManifest = buildI18nManifest(metadata.i18n); + + const localesToValidate = i18n + ? i18n.locales.map((locale) => locale.code) + : [undefined]; + for (const locale of localesToValidate) { + const navigation = await resolveDocsNavigation({ + srcDir: sourceMirror.srcDir, + groups, + mounts, + i18n: metadata.i18n, + locale, + }); + const firstUnknownGroup = navigation.unknown[0]; + if (firstUnknownGroup) { + throw new Error( + `${firstUnknownGroup.urlPath} declares unknown group "${firstUnknownGroup.slug}"` + ); + } } await convertAllMdx({ @@ -864,6 +972,8 @@ export async function runGenerateCommand( outDir, product, groups, + i18n: metadata.i18n, + locale: i18n?.defaultLocale, }); result = { docsDir, @@ -878,7 +988,11 @@ export async function runGenerateCommand( srcDir, }; } else { + if (i18n) { + await copyDefaultLocaleMarkdownAliases(outDir, i18n.defaultLocale); + } await copyMountedMarkdownMirrors(outDir, mounts); + const i18nManifestPath = await writeI18nManifest(outDir, i18nManifest); await generateLlmsTxt({ srcDir: sourceMirror.srcDir, outDir, @@ -886,6 +1000,8 @@ export async function runGenerateCommand( product, groups, mounts, + i18n: metadata.i18n, + locale: i18n?.defaultLocale, }); await generateLLMFullContextFiles({ @@ -894,12 +1010,16 @@ export async function runGenerateCommand( product: { name: product.name }, groups, mounts, + i18n: metadata.i18n, + locale: i18n?.defaultLocale, }); const search = await generateDocsSearchFiles({ outDir, baseUrl: args.baseUrl, mounts, + i18n: metadata.i18n, + locale: i18n?.defaultLocale, }); const agentReadability = await generateAgentReadabilityArtifacts({ outDir, @@ -907,8 +1027,55 @@ export async function runGenerateCommand( product, groups, mounts, + i18n: metadata.i18n, + locale: i18n?.defaultLocale, + i18nManifest, }); + if (i18n) { + for (const locale of i18n.locales) { + if (locale.code === i18n.defaultLocale) { + continue; + } + await generateLlmsTxt({ + srcDir: sourceMirror.srcDir, + outDir, + baseUrl: args.baseUrl, + product, + groups, + mounts, + i18n: metadata.i18n, + locale: locale.code, + }); + await generateLLMFullContextFiles({ + outDir, + baseUrl: args.baseUrl, + product: { name: product.name }, + groups, + mounts, + i18n: metadata.i18n, + locale: locale.code, + }); + await generateDocsSearchFiles({ + outDir, + baseUrl: args.baseUrl, + mounts, + i18n: metadata.i18n, + locale: locale.code, + }); + await generateAgentReadabilityArtifacts({ + outDir, + baseUrl: args.baseUrl, + product, + groups, + mounts, + i18n: metadata.i18n, + locale: locale.code, + i18nManifest, + }); + } + } + result = { docsDir, docsDirs, @@ -917,6 +1084,7 @@ export async function runGenerateCommand( docsRobotsTxt: agentReadability.files.robotsTxt, docsSitemapMd: agentReadability.files.sitemapMd, docsSitemapXml: agentReadability.files.sitemapXml, + i18nManifest: i18nManifestPath, docsLlmsTxt: path.join(outDir, "docs", "llms.txt"), llmsFullTxt: path.join(outDir, "llms-full.txt"), llmsTxt: path.join(outDir, "llms.txt"), diff --git a/packages/leadtype/src/i18n/i18n.test.ts b/packages/leadtype/src/i18n/i18n.test.ts new file mode 100644 index 0000000..bd8ff59 --- /dev/null +++ b/packages/leadtype/src/i18n/i18n.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { + getAlternateLocaleLinks, + getDocsLocaleUrlPrefix, + isDefaultLocale, + listDocsLocales, + logicalPathFromLocaleRelativePath, + normalizeDocsI18nConfig, + outputRelativePathForLocale, + resolveDocsLocale, + stripLocaleFromDocsPath, + toLocalizedDocsUrlPath, + toLocalizedMarkdownUrlPath, +} from "./index"; + +const i18n = { + defaultLocale: "en", + locales: ["en", "zh", { code: "fr", label: "Français" }], +} as const; + +describe("i18n helpers", () => { + it("normalizes shorthand and object locale entries", () => { + expect(normalizeDocsI18nConfig(i18n)).toMatchObject({ + defaultLocale: "en", + fallback: "default", + locales: [ + { code: "en" }, + { code: "zh" }, + { code: "fr", label: "Français" }, + ], + }); + }); + + it("validates locale config mistakes", () => { + expect(() => + normalizeDocsI18nConfig({ defaultLocale: "en", locales: ["en", "en"] }) + ).toThrow(/Duplicate locale code/); + expect(() => + normalizeDocsI18nConfig({ defaultLocale: "en", locales: ["zh"] }) + ).toThrow(/must be included/); + expect(() => + normalizeDocsI18nConfig({ defaultLocale: "en", locales: ["../en"] }) + ).toThrow(/Invalid locale code/); + }); + + it("returns empty locale lists when i18n is disabled", () => { + expect(normalizeDocsI18nConfig()).toBeUndefined(); + expect(listDocsLocales()).toEqual([]); + expect(resolveDocsLocale("/docs/quickstart")).toBeUndefined(); + expect(stripLocaleFromDocsPath("/docs/zh/quickstart")).toBe( + "/docs/zh/quickstart" + ); + expect( + getAlternateLocaleLinks( + { urlPath: "/docs/quickstart" }, + new Map([["en", { urlPath: "/docs/quickstart" }]]) + ) + ).toEqual([]); + }); + + it("keeps default docs unprefixed and prefixes translated locales", () => { + expect(toLocalizedDocsUrlPath("quickstart.mdx", "en", i18n)).toBe( + "/docs/quickstart" + ); + expect(toLocalizedDocsUrlPath("quickstart.mdx", "zh", i18n)).toBe( + "/docs/zh/quickstart" + ); + expect(toLocalizedMarkdownUrlPath("guides/index.md", "zh", i18n)).toBe( + "/docs/zh/guides/index.md" + ); + expect( + toLocalizedDocsUrlPath("v1.mdx", "zh", i18n, [ + { pathPrefix: "changelog", urlPrefix: "/changelog" }, + { pathPrefix: "", urlPrefix: "/docs" }, + ]) + ).toBe("/docs/zh/v1"); + expect( + toLocalizedDocsUrlPath("changelog/v1.mdx", "zh", i18n, [ + { pathPrefix: "changelog", urlPrefix: "/changelog" }, + { pathPrefix: "", urlPrefix: "/docs" }, + ]) + ).toBe("/changelog/zh/v1"); + }); + + it("exposes locale prefix helpers", () => { + expect(isDefaultLocale("en", i18n)).toBe(true); + expect(isDefaultLocale("zh", i18n)).toBe(false); + expect(getDocsLocaleUrlPrefix("en", i18n)).toBe("/docs"); + expect(getDocsLocaleUrlPrefix("zh", i18n)).toBe("/docs/zh"); + expect(getDocsLocaleUrlPrefix("zh", i18n, "/reference")).toBe( + "/reference/zh" + ); + }); + + it("resolves and strips locale prefixes from docs URLs", () => { + expect(resolveDocsLocale("/docs/quickstart", i18n)).toBe("en"); + expect(resolveDocsLocale("/docs/zh/quickstart", i18n)).toBe("zh"); + expect(resolveDocsLocale("/blog/zh/quickstart", i18n)).toBeUndefined(); + expect(stripLocaleFromDocsPath("/docs/zh/quickstart", i18n)).toBe( + "/docs/quickstart" + ); + expect(stripLocaleFromDocsPath("/docs/quickstart", i18n)).toBe( + "/docs/quickstart" + ); + expect(resolveDocsLocale("/reference/fr/intro", i18n, "/reference")).toBe( + "fr" + ); + }); + + it("derives logical paths and output paths from locale folders", () => { + const localeCodes = new Set(["en", "zh"]); + expect( + logicalPathFromLocaleRelativePath("zh/guides/setup.mdx", localeCodes) + ).toEqual({ + logicalPath: "guides/setup", + sourceLocale: "zh", + }); + expect( + logicalPathFromLocaleRelativePath("guides/setup.mdx", localeCodes) + ).toEqual({ + logicalPath: "guides/setup", + }); + expect(outputRelativePathForLocale("guides/setup.mdx", "en", i18n)).toBe( + "guides/setup" + ); + expect(outputRelativePathForLocale("guides/setup.mdx", "zh", i18n)).toBe( + "zh/guides/setup" + ); + }); + + it("returns alternate locale links from localized page maps", () => { + const alternates = getAlternateLocaleLinks( + { urlPath: "/docs/quickstart", logicalPath: "quickstart" }, + new Map([ + ["en", { urlPath: "/docs/quickstart", sourceLocale: "en" }], + ["zh", { urlPath: "/docs/zh/quickstart", sourceLocale: "en" }], + ]), + i18n + ); + + expect(alternates).toEqual([ + { + locale: "en", + urlPath: "/docs/quickstart", + isFallback: false, + }, + { + locale: "zh", + urlPath: "/docs/zh/quickstart", + isFallback: true, + }, + ]); + }); + + it("filters alternate links by logical path when available", () => { + const alternates = getAlternateLocaleLinks( + { urlPath: "/docs/quickstart", logicalPath: "quickstart" }, + new Map([ + [ + "en", + { + urlPath: "/docs/other", + logicalPath: "other", + sourceLocale: "en", + }, + ], + ]), + i18n + ); + + expect(alternates).toEqual([]); + }); +}); diff --git a/packages/leadtype/src/i18n/index.ts b/packages/leadtype/src/i18n/index.ts new file mode 100644 index 0000000..dd9d61d --- /dev/null +++ b/packages/leadtype/src/i18n/index.ts @@ -0,0 +1,318 @@ +import { + type DocsPathMount, + normalizeDocsPath, + normalizeUrlPrefix, + stripDocsExtension, + toDocsUrlPath, +} from "../internal/docs-url"; + +const LOCALE_CODE_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/; +const MARKDOWN_EXTENSION_PATTERN = /\.(md|mdx)$/; + +export type LocaleCode = string; + +export type DocsLocale = { + code: LocaleCode; + label?: string; + dir?: "ltr" | "rtl"; +}; + +export type DocsLocaleInput = LocaleCode | DocsLocale; + +export type DocsI18nConfig = { + defaultLocale: LocaleCode; + locales: DocsLocaleInput[]; + fallback?: "default"; +}; + +export type NormalizedDocsI18nConfig = { + defaultLocale: LocaleCode; + locales: DocsLocale[]; + fallback: "default"; +}; + +export type LocalizedDocsMetadata = { + locale?: LocaleCode; + sourceLocale?: LocaleCode; + isFallback?: boolean; + logicalPath?: string; +}; + +export type DocsLocaleArtifactPaths = { + locale: LocaleCode; + urlPrefix: string; + llmsTxt?: string; + llmsFullTxt?: string; + searchIndex?: string; + searchContent?: string; + agentReadabilityManifest?: string; + robotsTxt?: string; + sitemapMd?: string; + sitemapXml?: string; +}; + +export type DocsI18nManifest = { + version: 1; + defaultLocale: LocaleCode; + locales: DocsLocale[]; + artifacts: DocsLocaleArtifactPaths[]; +}; + +export type AlternateLocaleLink = { + locale: LocaleCode; + urlPath: string; + label?: string; + dir?: "ltr" | "rtl"; + isFallback: boolean; +}; + +export type LocalizedPageLike = { + locale?: LocaleCode; + sourceLocale?: LocaleCode; + isFallback?: boolean; + logicalPath?: string; + urlPath: string; +}; + +function assertValidLocaleCode(code: string): LocaleCode { + if (!LOCALE_CODE_PATTERN.test(code)) { + throw new Error( + `Invalid locale code "${code}". Locale codes must be URL-safe and may contain letters, numbers, underscores, or dashes.` + ); + } + return code; +} + +function normalizeLocale(input: DocsLocaleInput): DocsLocale { + if (typeof input === "string") { + return { code: assertValidLocaleCode(input) }; + } + return { + ...input, + code: assertValidLocaleCode(input.code), + }; +} + +export function normalizeDocsI18nConfig( + config?: DocsI18nConfig +): NormalizedDocsI18nConfig | undefined { + if (!config) { + return; + } + + const defaultLocale = assertValidLocaleCode(config.defaultLocale); + const seen = new Set(); + const locales: DocsLocale[] = []; + + for (const localeInput of config.locales) { + const locale = normalizeLocale(localeInput); + const key = locale.code.toLowerCase(); + if (seen.has(key)) { + throw new Error(`Duplicate locale code "${locale.code}" in i18n config.`); + } + seen.add(key); + locales.push(locale); + } + + if (!seen.has(defaultLocale.toLowerCase())) { + throw new Error( + `i18n.defaultLocale "${defaultLocale}" must be included in i18n.locales.` + ); + } + + return { + defaultLocale, + locales, + fallback: config.fallback ?? "default", + }; +} + +export function listDocsLocales(i18n?: DocsI18nConfig): DocsLocale[] { + return normalizeDocsI18nConfig(i18n)?.locales ?? []; +} + +export function isDefaultLocale( + locale: LocaleCode, + i18n?: DocsI18nConfig +): boolean { + return normalizeDocsI18nConfig(i18n)?.defaultLocale === locale; +} + +export function getDocsLocaleUrlPrefix( + locale: LocaleCode, + i18n?: DocsI18nConfig, + docsUrlPrefix = "/docs" +): string { + const normalized = normalizeDocsI18nConfig(i18n); + const prefix = normalizeUrlPrefix(docsUrlPrefix); + if (!normalized || locale === normalized.defaultLocale) { + return prefix; + } + return `${prefix}/${locale}`; +} + +function splitUrlPath(pathname: string): string[] { + return normalizeDocsPath(pathname) + .split("/") + .filter((segment) => segment.length > 0); +} + +export function resolveDocsLocale( + pathname: string, + i18n?: DocsI18nConfig, + docsUrlPrefix = "/docs" +): LocaleCode | undefined { + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized) { + return; + } + + const prefixSegments = splitUrlPath(docsUrlPrefix); + const pathnameSegments = splitUrlPath(pathname); + const isUnderDocsPrefix = prefixSegments.every( + (segment, index) => pathnameSegments[index] === segment + ); + if (!isUnderDocsPrefix) { + return; + } + + const afterPrefix = pathnameSegments.slice(prefixSegments.length); + const first = afterPrefix[0]; + const matched = normalized.locales.find((locale) => locale.code === first); + return matched?.code ?? normalized.defaultLocale; +} + +export function stripLocaleFromDocsPath( + pathname: string, + i18n?: DocsI18nConfig, + docsUrlPrefix = "/docs" +): string { + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized) { + return pathname; + } + + const prefix = normalizeUrlPrefix(docsUrlPrefix); + const segments = splitUrlPath(pathname); + const prefixSegments = splitUrlPath(prefix); + const localeCodes = new Set(normalized.locales.map((locale) => locale.code)); + + if ( + prefixSegments.every((segment, index) => segments[index] === segment) && + localeCodes.has(segments[prefixSegments.length] ?? "") + ) { + const stripped = [ + ...prefixSegments, + ...segments.slice(prefixSegments.length + 1), + ]; + return `/${stripped.join("/")}`; + } + + return pathname; +} + +export function toLocalizedDocsUrlPath( + relativePath: string, + locale: LocaleCode, + i18n?: DocsI18nConfig, + mounts?: DocsPathMount[] +): string { + const basePath = toDocsUrlPath(relativePath, mounts); + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized || locale === normalized.defaultLocale) { + return basePath; + } + + const matchedMount = [...(mounts ?? [{ pathPrefix: "", urlPrefix: "/docs" }])] + .map((mount) => normalizeUrlPrefix(mount.urlPrefix)) + .sort((left, right) => right.length - left.length) + .find( + (urlPrefix) => + basePath === urlPrefix || basePath.startsWith(`${urlPrefix}/`) + ); + const urlPrefix = matchedMount ?? "/docs"; + const suffix = basePath === urlPrefix ? "" : basePath.slice(urlPrefix.length); + return `${urlPrefix}/${locale}${suffix}`; +} + +export function toLocalizedMarkdownUrlPath( + relativePath: string, + locale: LocaleCode, + i18n?: DocsI18nConfig, + mounts?: DocsPathMount[] +): string { + const logicalPath = stripDocsExtension(relativePath); + const urlPath = toLocalizedDocsUrlPath( + `${logicalPath}.md`, + locale, + i18n, + mounts + ); + return logicalPath === "index" || logicalPath.endsWith("/index") + ? `${urlPath}/index.md` + : `${urlPath}.md`; +} + +export function logicalPathFromLocaleRelativePath( + relativePath: string, + localeCodes: Set +): { logicalPath: string; sourceLocale?: LocaleCode } { + const normalized = normalizeDocsPath(relativePath); + const segments = normalized.split("/"); + const first = segments[0] ?? ""; + if (localeCodes.has(first)) { + return { + logicalPath: stripDocsExtension(segments.slice(1).join("/")), + sourceLocale: first, + }; + } + return { + logicalPath: stripDocsExtension(normalized), + }; +} + +export function outputRelativePathForLocale( + logicalPath: string, + locale: LocaleCode, + i18n?: DocsI18nConfig +): string { + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized || locale === normalized.defaultLocale) { + return logicalPath.replace(MARKDOWN_EXTENSION_PATTERN, ""); + } + return `${locale}/${logicalPath.replace(MARKDOWN_EXTENSION_PATTERN, "")}`; +} + +export function getAlternateLocaleLinks( + page: LocalizedPageLike, + pagesByLocale: Map, + i18n?: DocsI18nConfig +): AlternateLocaleLink[] { + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized) { + return []; + } + + return normalized.locales.flatMap((locale) => { + const alternate = pagesByLocale.get(locale.code); + if (!alternate) { + return []; + } + if ( + page.logicalPath && + alternate.logicalPath && + page.logicalPath !== alternate.logicalPath + ) { + return []; + } + return [ + { + locale: locale.code, + urlPath: alternate.urlPath, + isFallback: alternate.sourceLocale !== locale.code, + ...(locale.label ? { label: locale.label } : {}), + ...(locale.dir ? { dir: locale.dir } : {}), + }, + ]; + }); +} diff --git a/packages/leadtype/src/index.ts b/packages/leadtype/src/index.ts index 6df59e5..e748c6d 100644 --- a/packages/leadtype/src/index.ts +++ b/packages/leadtype/src/index.ts @@ -8,6 +8,22 @@ // - `leadtype/llm` — TOC extraction, slug helpers, agent readability // - `leadtype/search` — search index + per-host adapters // - `leadtype/lint` — frontmatter / meta.json validation + +export { + type AlternateLocaleLink, + type DocsI18nConfig, + type DocsI18nManifest, + type DocsLocale, + type DocsLocaleArtifactPaths, + getAlternateLocaleLinks, + type LocaleCode, + listDocsLocales, + normalizeDocsI18nConfig, + resolveDocsLocale, + stripLocaleFromDocsPath, + toLocalizedDocsUrlPath, + toLocalizedMarkdownUrlPath, +} from "./i18n"; export { type AgentReadabilityConfig, type AgentReadabilityManifest, diff --git a/packages/leadtype/src/internal/package-surface.test.ts b/packages/leadtype/src/internal/package-surface.test.ts index d43390d..0f0227f 100644 --- a/packages/leadtype/src/internal/package-surface.test.ts +++ b/packages/leadtype/src/internal/package-surface.test.ts @@ -10,6 +10,7 @@ describe("package surface", () => { ".", "./mdx", "./fumadocs", + "./i18n", "./remark", "./convert", "./llm", diff --git a/packages/leadtype/src/llm/index.ts b/packages/leadtype/src/llm/index.ts index 34a2de3..24516be 100644 --- a/packages/leadtype/src/llm/index.ts +++ b/packages/leadtype/src/llm/index.ts @@ -1,4 +1,19 @@ // Build-time exports. + +export { + type AlternateLocaleLink, + type DocsI18nConfig, + type DocsI18nManifest, + type DocsLocale, + getAlternateLocaleLinks, + type LocaleCode, + listDocsLocales, + normalizeDocsI18nConfig, + resolveDocsLocale, + stripLocaleFromDocsPath, + toLocalizedDocsUrlPath, + toLocalizedMarkdownUrlPath, +} from "../i18n"; export { type AgentReadabilityConfig, type AgentReadabilityResult, diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index c6ec301..0602049 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -205,6 +205,74 @@ describe("generateLlmsTxt", () => { expect(searchSection).toContain("Search Only"); expect(selfHostSection).not.toContain("Search Only"); }); + + it("renders locale-scoped llms summaries without fallback pages", async () => { + const projectDir = await createTempProject(); + const outDir = path.join(projectDir, "out"); + + await seedDocs(projectDir, [ + { + relativePath: "quickstart.mdx", + frontmatter: + "title: Quickstart\ndescription: English quickstart.\ngroup: get-started", + }, + { + relativePath: "setup.mdx", + frontmatter: + "title: Setup\ndescription: English setup.\ngroup: get-started", + }, + { + relativePath: "zh/quickstart.mdx", + frontmatter: + "title: 快速开始\ndescription: 中文快速开始。\ngroup: get-started", + }, + ]); + + await generateLlmsTxt({ + srcDir: projectDir, + outDir, + baseUrl: "https://leadtype.dev", + product: { name: "Leadtype", summary: "Docs pipeline." }, + groups: [{ slug: "get-started", title: "Get Started" }], + i18n: { defaultLocale: "en", locales: ["en", "zh"] }, + locale: "zh", + }); + + const zhSummary = await readFile( + path.join(outDir, "docs", "zh", "llms.txt"), + "utf8" + ); + expect(zhSummary).toContain("快速开始"); + expect(zhSummary).toContain("](/docs/zh/quickstart.md)"); + expect(zhSummary).not.toContain("Setup"); + }); + + it("rejects duplicate localized source files for the same locale and logical path", async () => { + const projectDir = await createTempProject(); + const outDir = path.join(projectDir, "out"); + + await seedDocs(projectDir, [ + { + relativePath: "quickstart.md", + frontmatter: "title: Quickstart", + }, + { + relativePath: "quickstart.mdx", + frontmatter: "title: Quickstart duplicate", + }, + ]); + + await expect( + generateLlmsTxt({ + srcDir: projectDir, + outDir, + baseUrl: "https://leadtype.dev", + product: { name: "Leadtype" }, + i18n: { defaultLocale: "en", locales: ["en", "zh"] }, + locale: "en", + }) + ).rejects.toThrow(/Duplicate docs file.*locale "en"/); + }); }); describe("generateLLMFullContextFiles", () => { @@ -302,6 +370,47 @@ describe("generateLLMFullContextFiles", () => { expect(llmsFull).toContain("Shared body."); }); + it("writes non-default locale full-context files under the locale docs path", async () => { + const projectDir = await createTempProject(); + await seedDocs(projectDir, [ + { + relativePath: "quickstart.md", + frontmatter: + "title: Quickstart\ndescription: English quickstart.\ngroup: get-started", + body: "# Quickstart\n\nEnglish body.\n", + }, + { + relativePath: "setup.md", + frontmatter: + "title: Setup\ndescription: English setup.\ngroup: get-started", + body: "# Setup\n\nEnglish setup.\n", + }, + { + relativePath: "zh/quickstart.md", + frontmatter: + "title: 快速开始\ndescription: 中文快速开始。\ngroup: get-started", + body: "# 快速开始\n\n中文正文。\n", + }, + ]); + + await generateLLMFullContextFiles({ + outDir: projectDir, + baseUrl: "https://leadtype.dev", + product: { name: "Leadtype" }, + groups: [{ slug: "get-started", title: "Get Started" }], + i18n: { defaultLocale: "en", locales: ["en", "zh"] }, + locale: "zh", + }); + + const llmsFull = await readFile( + path.join(projectDir, "docs", "zh", "llms-full.txt"), + "utf8" + ); + expect(llmsFull).toContain("快速开始"); + expect(llmsFull).toContain("https://leadtype.dev/docs/zh/quickstart"); + expect(llmsFull).not.toContain("English setup"); + }); + it("clears stale docs-scoped full-context files", async () => { const projectDir = await createTempProject(); await mkdir(path.join(projectDir, "docs", "llms-full", "frameworks"), { diff --git a/packages/leadtype/src/llm/llm.ts b/packages/leadtype/src/llm/llm.ts index b25e4dd..f3bf82a 100644 --- a/packages/leadtype/src/llm/llm.ts +++ b/packages/leadtype/src/llm/llm.ts @@ -8,6 +8,16 @@ import { writeFile, } from "node:fs/promises"; import path from "node:path"; +import { + type DocsI18nConfig, + type DocsI18nManifest, + type LocaleCode, + type LocalizedDocsMetadata, + logicalPathFromLocaleRelativePath, + normalizeDocsI18nConfig, + outputRelativePathForLocale, + toLocalizedDocsUrlPath, +} from "../i18n"; import { slugifyDocsHeading } from "../internal/docs-heading"; import { type DocsPathMount, @@ -61,11 +71,10 @@ function assertValidGroupSlug(slug: string, parentPath: string[]): string { } return slug; } -const MD_ONLY_EXTENSION_PATTERN = /\.md$/; const GENERATED_MARKDOWN_FILES = new Set([SITEMAP_MARKDOWN_FILE]); const SEPARATOR_PATTERN = /[-_]/; -export type SourceDoc = { +export type SourceDoc = LocalizedDocsMetadata & { title: string; description: string; urlPath: string; @@ -138,6 +147,7 @@ export type DocsGroup = { export type DocsConfig = { product: ProductInfo; groups: DocsGroup[]; + i18n?: DocsI18nConfig; /** * Optional base directory for ExtractedTypeTable / AutoTypeTable path * resolution during generation. Relative values are resolved from `--src`. @@ -164,6 +174,8 @@ export type LlmsTxtConfig = { groups: DocsGroup[]; /** Optional path-to-URL mounts for generated docs, e.g. changelog -> /changelog. */ mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; }; export type LLMFullContextConfig = { @@ -173,6 +185,8 @@ export type LLMFullContextConfig = { /** Group tree from `docs.config.ts`. Preserved for config validation. */ groups: DocsGroup[]; mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; }; export type AgentReadabilityConfig = { @@ -181,6 +195,9 @@ export type AgentReadabilityConfig = { product: Pick; groups: DocsGroup[]; mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; + i18nManifest?: DocsI18nManifest; }; export type AgentReadabilityResult = { @@ -198,6 +215,9 @@ export type ResolveDocsNavigationConfig = { baseUrl?: string; groups: DocsGroup[]; mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; + includeFallback?: boolean; toc?: boolean | DocsTableOfContentsOptions; /** * Name of the docs subdirectory under `srcDir`. Defaults to `"docs"` for @@ -212,6 +232,8 @@ export type ResolveDocsTableOfContentsConfig = { srcDir: string; baseUrl?: string; mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; options?: DocsTableOfContentsOptions; }; @@ -544,11 +566,135 @@ async function collectFiles( return files.flat(); } +type LocaleReadOptions = { + i18n?: DocsI18nConfig; + locale?: LocaleCode; + includeFallback?: boolean; +}; + +function resolveLocaleReadOptions(options: LocaleReadOptions): { + i18n: ReturnType; + locale?: LocaleCode; + includeFallback: boolean; +} { + const i18n = normalizeDocsI18nConfig(options.i18n); + if (!i18n) { + return { i18n, includeFallback: false }; + } + const locale = options.locale ?? i18n.defaultLocale; + if (!i18n.locales.some((entry) => entry.code === locale)) { + throw new Error(`Unknown locale "${locale}" in i18n config.`); + } + return { + i18n, + locale, + includeFallback: options.includeFallback ?? i18n.fallback === "default", + }; +} + +function assertUnambiguousDefaultLocaleLayout( + relativePaths: string[], + localeCodes: Set, + defaultLocale: string +): void { + const hasRootDefault = relativePaths.some((relativePath) => { + const first = normalizeDocsPath(relativePath).split("/")[0] ?? ""; + return !localeCodes.has(first); + }); + const hasDefaultFolder = relativePaths.some((relativePath) => + normalizeDocsPath(relativePath).startsWith(`${defaultLocale}/`) + ); + + if (hasRootDefault && hasDefaultFolder) { + throw new Error( + `Ambiguous i18n default-locale layout. Use either root docs files or docs/${defaultLocale}/ files for the default locale, not both.` + ); + } +} + +type SelectedDocFile = LocalizedDocsMetadata & { + filePath: string; + logicalPath: string; + outputRelativePath: string; + sourceLocale?: LocaleCode; +}; + +function selectLocalizedFiles( + files: string[], + docsDir: string, + options: { + defaultLocale: LocaleCode; + locale: LocaleCode; + localeCodes: Set; + includeFallback: boolean; + } +): SelectedDocFile[] { + const byLogicalPath = new Map< + string, + Map + >(); + + for (const filePath of files) { + const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); + const { logicalPath, sourceLocale } = logicalPathFromLocaleRelativePath( + relativePath, + options.localeCodes + ); + const resolvedSourceLocale = sourceLocale ?? options.defaultLocale; + const localeFiles = byLogicalPath.get(logicalPath) ?? new Map(); + const existing = localeFiles.get(resolvedSourceLocale); + if (existing) { + throw new Error( + `Duplicate docs file for logical path "${logicalPath}" and locale "${resolvedSourceLocale}": "${existing.filePath}" conflicts with "${filePath}". Rename one or remove it.` + ); + } + + localeFiles.set(resolvedSourceLocale, { + filePath, + sourceLocale: resolvedSourceLocale, + }); + byLogicalPath.set(logicalPath, localeFiles); + } + + const selected: SelectedDocFile[] = []; + for (const [logicalPath, localeFiles] of byLogicalPath) { + const localized = localeFiles.get(options.locale); + const fallback = + options.includeFallback && options.locale !== options.defaultLocale + ? localeFiles.get(options.defaultLocale) + : undefined; + const match = localized ?? fallback; + if (!match) { + continue; + } + selected.push({ + filePath: match.filePath, + locale: options.locale, + sourceLocale: match.sourceLocale, + isFallback: match.sourceLocale !== options.locale, + logicalPath, + outputRelativePath: outputRelativePathForLocale( + logicalPath, + options.locale, + { + defaultLocale: options.defaultLocale, + locales: Array.from(options.localeCodes), + } + ), + }); + } + + return selected.sort((left, right) => + left.outputRelativePath.localeCompare(right.outputRelativePath) + ); +} + async function readSourceDocs( srcDir: string, baseUrl: string, mounts?: DocsPathMount[], - docsDirName: string = DOCS_DIRNAME + docsDirName: string = DOCS_DIRNAME, + localeOptions: LocaleReadOptions = {} ): Promise> { const docsDir = path.join(srcDir, docsDirName); const docs = new Map(); @@ -558,23 +704,67 @@ async function readSourceDocs( } const files = await collectFiles(docsDir, [".md", ".mdx"]); + const relativePaths = files.map((filePath) => + normalizeDocsPath(path.relative(docsDir, filePath)) + ); + const localeRead = resolveLocaleReadOptions(localeOptions); + const localeCodes = new Set( + localeRead.i18n?.locales.map((locale) => locale.code) ?? [] + ); + + if (localeRead.i18n && localeRead.locale) { + assertUnambiguousDefaultLocaleLayout( + relativePaths, + localeCodes, + localeRead.i18n.defaultLocale + ); + } + + const selectedFiles: SelectedDocFile[] = + localeRead.i18n && localeRead.locale + ? selectLocalizedFiles(files, docsDir, { + defaultLocale: localeRead.i18n.defaultLocale, + locale: localeRead.locale, + localeCodes, + includeFallback: localeRead.includeFallback, + }) + : files.map((filePath) => { + const relativePath = normalizeDocsPath( + path.relative(docsDir, filePath) + ); + return { + filePath, + logicalPath: stripDocsExtension(relativePath), + outputRelativePath: stripDocsExtension(relativePath), + }; + }); const entries = await Promise.all( - files.map(async (filePath) => { - const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); - const raw = await readFile(filePath, "utf-8"); + selectedFiles.map(async (file) => { + const relativePath = normalizeDocsPath( + path.relative(docsDir, file.filePath) + ); + const raw = await readFile(file.filePath, "utf-8"); const parsed = parseFrontmatter(raw); const title = String(parsed.data.title ?? "").trim() || titleFromRelativePath( - relativePath, + `${file.logicalPath}${path.extname(relativePath)}`, path.extname(relativePath) as ".md" | ".mdx" ) || "Untitled"; const description = normalizeDescription( String(parsed.data.description ?? "") ); - const urlPath = toUrlPath(relativePath, mounts); + const urlPath = + localeRead.i18n && localeRead.locale + ? toLocalizedDocsUrlPath( + `${file.logicalPath}.mdx`, + localeRead.locale, + localeRead.i18n, + mounts + ) + : toUrlPath(relativePath, mounts); const groups = normalizeGroupValue(parsed.data.group); const orderRaw = parsed.data.order; const order = @@ -588,9 +778,15 @@ async function readSourceDocs( description, urlPath, absoluteUrl: toAbsoluteUrl(urlPath, baseUrl), - relativePath: stripDocsExtension(relativePath), + relativePath: file.outputRelativePath, groups, ...(order === undefined ? {} : { order }), + ...(file.locale ? { locale: file.locale } : {}), + ...(file.sourceLocale ? { sourceLocale: file.sourceLocale } : {}), + ...(file.isFallback === undefined + ? {} + : { isFallback: file.isFallback }), + ...(file.logicalPath ? { logicalPath: file.logicalPath } : {}), content: parsed.content, }, }; @@ -613,7 +809,8 @@ async function readSourceDocs( async function readMarkdownDocs( outDir: string, baseUrl: string, - mounts?: DocsPathMount[] + mounts?: DocsPathMount[], + localeOptions: LocaleReadOptions = {} ): Promise { const docsDir = path.join(outDir, DOCS_DIRNAME); if (!existsSync(docsDir)) { @@ -621,20 +818,62 @@ async function readMarkdownDocs( } const files = await collectFiles(docsDir, [".md"]); + const relativePaths = files.map((filePath) => + normalizeDocsPath(path.relative(docsDir, filePath)) + ); + const localeRead = resolveLocaleReadOptions(localeOptions); + const localeCodes = new Set( + localeRead.i18n?.locales.map((locale) => locale.code) ?? [] + ); + if (localeRead.i18n && localeRead.locale) { + assertUnambiguousDefaultLocaleLayout( + relativePaths, + localeCodes, + localeRead.i18n.defaultLocale + ); + } + const selectedFiles: SelectedDocFile[] = + localeRead.i18n && localeRead.locale + ? selectLocalizedFiles(files, docsDir, { + defaultLocale: localeRead.i18n.defaultLocale, + locale: localeRead.locale, + localeCodes, + includeFallback: localeRead.includeFallback, + }) + : files.map((filePath) => { + const relativePath = normalizeDocsPath( + path.relative(docsDir, filePath) + ); + return { + filePath, + logicalPath: stripDocsExtension(relativePath), + outputRelativePath: stripDocsExtension(relativePath), + }; + }); const docs = await Promise.all( - files.map(async (filePath) => { - const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); - const raw = await readFile(filePath, "utf-8"); - const fileStat = await stat(filePath); + selectedFiles.map(async (file) => { + const relativePath = normalizeDocsPath( + path.relative(docsDir, file.filePath) + ); + const raw = await readFile(file.filePath, "utf-8"); + const fileStat = await stat(file.filePath); const parsed = parseFrontmatter(raw); const title = String(parsed.data.title ?? "").trim() || - titleFromRelativePath(relativePath, ".md") || + titleFromRelativePath(`${file.logicalPath}.md`, ".md") || "Untitled"; const description = normalizeDescription( String(parsed.data.description ?? "") ); - const urlPath = toUrlPath(relativePath, mounts); + const urlPath = + localeRead.i18n && localeRead.locale + ? toLocalizedDocsUrlPath( + `${file.logicalPath}.md`, + localeRead.locale, + localeRead.i18n, + mounts + ) + : toUrlPath(relativePath, mounts); const groups = normalizeGroupValue(parsed.data.group); return { @@ -642,8 +881,14 @@ async function readMarkdownDocs( description, urlPath, absoluteUrl: toAbsoluteUrl(urlPath, baseUrl), - relativePath: relativePath.replace(MD_ONLY_EXTENSION_PATTERN, ""), + relativePath: file.outputRelativePath, groups, + ...(file.locale ? { locale: file.locale } : {}), + ...(file.sourceLocale ? { sourceLocale: file.sourceLocale } : {}), + ...(file.isFallback === undefined + ? {} + : { isFallback: file.isFallback }), + ...(file.logicalPath ? { logicalPath: file.logicalPath } : {}), content: parsed.content.trim(), lastModified: readLastModified(parsed.data, fileStat.mtime), }; @@ -651,7 +896,13 @@ async function readMarkdownDocs( ); return docs - .filter((doc) => !GENERATED_MARKDOWN_FILES.has(`${doc.relativePath}.md`)) + .filter((doc) => { + const filename = path.basename(`${doc.relativePath}.md`); + return !( + GENERATED_MARKDOWN_FILES.has(`${doc.relativePath}.md`) || + GENERATED_MARKDOWN_FILES.has(filename) + ); + }) .sort((left, right) => left.urlPath.localeCompare(right.urlPath)); } @@ -876,20 +1127,40 @@ export async function generateLlmsTxt(config: LlmsTxtConfig): Promise { const srcDir = path.resolve(config.srcDir); const outDir = path.resolve(config.outDir); const baseUrl = normalizeBaseUrl(config.baseUrl); - const sourceDocs = await readSourceDocs(srcDir, baseUrl, config.mounts); + const i18n = normalizeDocsI18nConfig(config.i18n); + const locale = config.locale ?? i18n?.defaultLocale; + const sourceDocs = await readSourceDocs( + srcDir, + baseUrl, + config.mounts, + DOCS_DIRNAME, + { + i18n: config.i18n, + locale, + includeFallback: false, + } + ); const resolved = resolveGroups(config.groups); const membership = buildGroupMembership([...sourceDocs.values()], resolved); await mkdir(path.join(outDir, DOCS_DIRNAME), { recursive: true }); - await writeFile( - path.join(outDir, "llms.txt"), - renderProductSummary(config.product, sourceDocs, config.mounts) - ); + const isDefaultLocale = !i18n || locale === i18n.defaultLocale; + if (isDefaultLocale) { + await writeFile( + path.join(outDir, "llms.txt"), + renderProductSummary(config.product, sourceDocs, config.mounts) + ); + } if (resolved.length > 0) { + const docsLlmsPath = + i18n && locale && locale !== i18n.defaultLocale + ? path.join(outDir, DOCS_DIRNAME, locale, "llms.txt") + : path.join(outDir, DOCS_DIRNAME, "llms.txt"); + await mkdir(path.dirname(docsLlmsPath), { recursive: true }); await writeFile( - path.join(outDir, DOCS_DIRNAME, "llms.txt"), + docsLlmsPath, renderDocsSummary(config.product, resolved, membership, config.mounts) ); } @@ -904,7 +1175,13 @@ export async function generateLLMFullContextFiles( ): Promise { const outDir = path.resolve(config.outDir); const baseUrl = normalizeBaseUrl(config.baseUrl); - const markdownDocs = await readMarkdownDocs(outDir, baseUrl, config.mounts); + const i18n = normalizeDocsI18nConfig(config.i18n); + const locale = config.locale ?? i18n?.defaultLocale; + const markdownDocs = await readMarkdownDocs(outDir, baseUrl, config.mounts, { + i18n: config.i18n, + locale, + includeFallback: false, + }); if (markdownDocs.length === 0) { throw new Error( @@ -914,11 +1191,26 @@ export async function generateLLMFullContextFiles( resolveGroups(config.groups); - const llmsFullDir = path.join(outDir, DOCS_DIRNAME, "llms-full"); - await rm(llmsFullDir, { recursive: true, force: true }); - await rm(path.join(outDir, DOCS_DIRNAME, "llms-full.txt"), { force: true }); + if (!i18n || locale === i18n.defaultLocale) { + const llmsFullDir = path.join(outDir, DOCS_DIRNAME, "llms-full"); + await rm(llmsFullDir, { recursive: true, force: true }); + await rm(path.join(outDir, DOCS_DIRNAME, "llms-full.txt"), { force: true }); + await writeFile( + path.join(outDir, "llms-full.txt"), + renderFullContextDocument(config.product, markdownDocs) + ); + return; + } + + const localeFullPath = path.join( + outDir, + DOCS_DIRNAME, + locale ?? i18n.defaultLocale, + "llms-full.txt" + ); + await mkdir(path.dirname(localeFullPath), { recursive: true }); await writeFile( - path.join(outDir, "llms-full.txt"), + localeFullPath, renderFullContextDocument(config.product, markdownDocs) ); } @@ -942,6 +1234,10 @@ function toAgentReadabilityPage( relativePath: doc.relativePath, groups: [...doc.groups], lastModified: doc.lastModified, + ...(doc.locale ? { locale: doc.locale } : {}), + ...(doc.sourceLocale ? { sourceLocale: doc.sourceLocale } : {}), + ...(doc.isFallback === undefined ? {} : { isFallback: doc.isFallback }), + ...(doc.logicalPath ? { logicalPath: doc.logicalPath } : {}), }; } @@ -965,6 +1261,7 @@ function buildNavigationFromMarkdownDocs( urlPath: page.urlPath, slug, })), + locale: docs[0]?.locale, }; } @@ -978,9 +1275,22 @@ export async function generateAgentReadabilityArtifacts( config: AgentReadabilityConfig ): Promise { const outDir = path.resolve(config.outDir); - const docsDir = path.join(outDir, DOCS_DIRNAME); const baseUrl = normalizeBaseUrl(config.baseUrl); - const markdownDocs = await readMarkdownDocs(outDir, baseUrl, config.mounts); + const i18n = normalizeDocsI18nConfig(config.i18n); + const locale = config.locale ?? i18n?.defaultLocale; + const docsDir = + i18n && locale && locale !== i18n.defaultLocale + ? path.join(outDir, DOCS_DIRNAME, locale) + : path.join(outDir, DOCS_DIRNAME); + const docsUrlPrefix = + i18n && locale && locale !== i18n.defaultLocale + ? `/docs/${locale}` + : "/docs"; + const markdownDocs = await readMarkdownDocs(outDir, baseUrl, config.mounts, { + i18n: config.i18n, + locale, + includeFallback: false, + }); if (markdownDocs.length === 0) { throw new Error( @@ -998,12 +1308,14 @@ export async function generateAgentReadabilityArtifacts( generatedAt: new Date().toISOString(), baseUrl, product: config.product, + ...(locale ? { locale } : {}), + ...(config.i18nManifest ? { i18n: config.i18nManifest } : {}), pages, navigation, files: { - robotsTxt: `/docs/${ROBOTS_FILE}`, - sitemapMd: `/docs/${SITEMAP_MARKDOWN_FILE}`, - sitemapXml: `/docs/${SITEMAP_XML_FILE}`, + robotsTxt: `${docsUrlPrefix}/${ROBOTS_FILE}`, + sitemapMd: `${docsUrlPrefix}/${SITEMAP_MARKDOWN_FILE}`, + sitemapXml: `${docsUrlPrefix}/${SITEMAP_XML_FILE}`, }, }; @@ -1028,7 +1340,7 @@ export async function generateAgentReadabilityArtifacts( files.robotsTxt, renderRobotsTxt({ baseUrl, - sitemapUrlPath: "/docs/sitemap.xml", + sitemapUrlPath: `${docsUrlPrefix}/sitemap.xml`, }) ); await writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`); @@ -1054,6 +1366,8 @@ export type AgentsMdConfig = { * Used for the relative-path prefix in every link. Default: `docs`. */ docsSubdir?: string; + i18n?: DocsI18nConfig; + locale?: LocaleCode; }; export type AgentsMdResult = { @@ -1088,7 +1402,17 @@ export async function generateAgentsMd( // field, but AGENTS.md output never reads that field — relative paths only. // Pass through any configured fallback so SourceDoc objects are well-formed. const baseUrl = normalizeBaseUrl(undefined); - const sourceDocs = await readSourceDocs(srcDir, baseUrl); + const sourceDocs = await readSourceDocs( + srcDir, + baseUrl, + undefined, + DOCS_DIRNAME, + { + i18n: config.i18n, + locale: config.locale, + includeFallback: false, + } + ); const resolved = resolveGroups(config.groups); const membership = buildGroupMembership([...sourceDocs.values()], resolved); @@ -1176,6 +1500,10 @@ function pageView( description: doc.description, groups: [...doc.groups], toc: tocByUrlPath.get(doc.urlPath) ?? [], + ...(doc.locale ? { locale: doc.locale } : {}), + ...(doc.sourceLocale ? { sourceLocale: doc.sourceLocale } : {}), + ...(doc.isFallback === undefined ? {} : { isFallback: doc.isFallback }), + ...(doc.logicalPath ? { logicalPath: doc.logicalPath } : {}), }; } @@ -1212,7 +1540,12 @@ export async function resolveDocsNavigation( srcDir, baseUrl, config.mounts, - config.docsDirName + config.docsDirName, + { + i18n: config.i18n, + locale: config.locale, + includeFallback: config.includeFallback ?? true, + } ); const resolved = resolveGroups(config.groups); const membership = buildGroupMembership([...sourceDocs.values()], resolved); @@ -1237,6 +1570,8 @@ export async function resolveDocsNavigation( urlPath: page.urlPath, slug, })), + locale: + config.locale ?? normalizeDocsI18nConfig(config.i18n)?.defaultLocale, }; } @@ -1245,7 +1580,17 @@ export async function resolveDocsTableOfContents( ): Promise { const srcDir = path.resolve(config.srcDir); const baseUrl = normalizeBaseUrl(config.baseUrl); - const sourceDocs = await readSourceDocs(srcDir, baseUrl, config.mounts); + const sourceDocs = await readSourceDocs( + srcDir, + baseUrl, + config.mounts, + DOCS_DIRNAME, + { + i18n: config.i18n, + locale: config.locale, + includeFallback: false, + } + ); return [...sourceDocs.values()] .sort((left, right) => left.urlPath.localeCompare(right.urlPath)) diff --git a/packages/leadtype/src/llm/readability.ts b/packages/leadtype/src/llm/readability.ts index 9ac0d4d..f81117f 100644 --- a/packages/leadtype/src/llm/readability.ts +++ b/packages/leadtype/src/llm/readability.ts @@ -6,6 +6,7 @@ * Cloudflare Workers, Hono, Astro, Nuxt, Vite middleware, etc. */ +import type { DocsI18nManifest, LocalizedDocsMetadata } from "../i18n"; import { normalizeDocsPath, stripTrailingSlashes, @@ -53,7 +54,7 @@ const XML_ESCAPE_PATTERN = /[<>&'"]/g; export type JsonLdValue = Record; -export type AgentReadabilityPage = { +export type AgentReadabilityPage = LocalizedDocsMetadata & { title: string; description: string; urlPath: string; @@ -80,7 +81,7 @@ export type DocsTableOfContentsOptions = { maxLevel?: 1 | 2 | 3 | 4 | 5 | 6; }; -export type DocsNavigationPage = { +export type DocsNavigationPage = LocalizedDocsMetadata & { urlPath: string; title: string; description: string; @@ -103,6 +104,7 @@ export type DocsNavigation = { ungrouped: DocsNavigationPage[]; /** Pages that named a group slug not present in the config. */ unknown: { urlPath: string; slug: string }[]; + locale?: string; }; export type AgentReadabilityManifest = { @@ -110,6 +112,8 @@ export type AgentReadabilityManifest = { generatedAt: string; baseUrl: string; product: { name: string; summary: string }; + locale?: string; + i18n?: DocsI18nManifest; pages: AgentReadabilityPage[]; navigation: DocsNavigation; files: { diff --git a/packages/leadtype/src/search/node.ts b/packages/leadtype/src/search/node.ts index 737d161..d56dc7a 100644 --- a/packages/leadtype/src/search/node.ts +++ b/packages/leadtype/src/search/node.ts @@ -1,6 +1,14 @@ import { existsSync } from "node:fs"; import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; +import { + type DocsI18nConfig, + type LocaleCode, + logicalPathFromLocaleRelativePath, + normalizeDocsI18nConfig, + outputRelativePathForLocale, + toLocalizedDocsUrlPath, +} from "../i18n"; import { type DocsPathMount, GENERIC_DOC_TITLES, @@ -32,6 +40,8 @@ export type GenerateDocsSearchFilesConfig = { outDir: string; baseUrl?: string; mounts?: DocsPathMount[]; + i18n?: DocsI18nConfig; + locale?: LocaleCode; outputFile?: string; contentOutputFile?: string; embedContent?: boolean; @@ -87,14 +97,23 @@ async function collectMarkdownFiles(rootDir: string): Promise { async function readMarkdownDocs( docsDir: string, baseUrl: string, - mounts?: DocsPathMount[] + mounts?: DocsPathMount[], + i18nConfig?: DocsI18nConfig, + requestedLocale?: LocaleCode ): Promise { const files = await collectMarkdownFiles(docsDir); + const i18n = normalizeDocsI18nConfig(i18nConfig); + const locale = requestedLocale ?? i18n?.defaultLocale; + const selectedFiles = selectMarkdownFiles(files, docsDir, i18nConfig, locale); const docs: DocsSearchDocument[] = []; - for (const filePath of files) { + for (const file of selectedFiles) { + const filePath = file.filePath; const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); - if (GENERATED_MARKDOWN_FILES.has(relativePath)) { + if ( + GENERATED_MARKDOWN_FILES.has(relativePath) || + GENERATED_MARKDOWN_FILES.has(path.basename(relativePath)) + ) { continue; } const raw = await readFile(filePath, "utf-8"); @@ -105,14 +124,20 @@ async function readMarkdownDocs( const description = normalizeDescription( String(parsed.data.description ?? "") ); - const urlPath = toDocsUrlPath(relativePath, mounts); + const urlPath = + i18n && locale + ? toLocalizedDocsUrlPath(`${file.logicalPath}.md`, locale, i18n, mounts) + : toDocsUrlPath(relativePath, mounts); docs.push({ - id: stripDocsExtension(relativePath), + id: urlPath, title, description, urlPath, absoluteUrl: toAbsoluteUrl(urlPath, baseUrl), - relativePath: stripDocsExtension(relativePath), + relativePath: file.outputRelativePath, + ...(file.locale ? { locale: file.locale } : {}), + ...(file.sourceLocale ? { sourceLocale: file.sourceLocale } : {}), + ...(file.logicalPath ? { logicalPath: file.logicalPath } : {}), content: parsed.content.trim(), }); } @@ -120,6 +145,69 @@ async function readMarkdownDocs( return docs; } +type SelectedMarkdownFile = { + filePath: string; + logicalPath: string; + outputRelativePath: string; + locale?: LocaleCode; + sourceLocale?: LocaleCode; +}; + +function selectMarkdownFiles( + files: string[], + docsDir: string, + i18nConfig?: DocsI18nConfig, + requestedLocale?: LocaleCode +): SelectedMarkdownFile[] { + const i18n = normalizeDocsI18nConfig(i18nConfig); + if (!(i18n && requestedLocale)) { + return files.map((filePath) => { + const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); + return { + filePath, + logicalPath: stripDocsExtension(relativePath), + outputRelativePath: stripDocsExtension(relativePath), + }; + }); + } + + const localeCodes = new Set(i18n.locales.map((entry) => entry.code)); + const byLogicalPath = new Map< + string, + Map + >(); + for (const filePath of files) { + const relativePath = normalizeDocsPath(path.relative(docsDir, filePath)); + const { logicalPath, sourceLocale } = logicalPathFromLocaleRelativePath( + relativePath, + localeCodes + ); + const resolvedLocale = sourceLocale ?? i18n.defaultLocale; + const localeFiles = byLogicalPath.get(logicalPath) ?? new Map(); + localeFiles.set(resolvedLocale, { + filePath, + logicalPath, + outputRelativePath: outputRelativePathForLocale( + logicalPath, + requestedLocale, + i18nConfig + ), + locale: requestedLocale, + sourceLocale: resolvedLocale, + }); + byLogicalPath.set(logicalPath, localeFiles); + } + + return Array.from(byLogicalPath.values()) + .flatMap((localeFiles) => { + const direct = localeFiles.get(requestedLocale); + return direct ? [direct] : []; + }) + .sort((left, right) => + left.outputRelativePath.localeCompare(right.outputRelativePath) + ); +} + function formatBytes(bytes: number): string { if (bytes >= 1024 * 1024) { return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; @@ -190,6 +278,12 @@ export async function generateDocsSearchFiles( ): Promise { const outDir = path.resolve(config.outDir); const docsDir = path.join(outDir, DOCS_DIRNAME); + const i18n = normalizeDocsI18nConfig(config.i18n); + const locale = config.locale ?? i18n?.defaultLocale; + const outputDocsDir = + i18n && locale && locale !== i18n.defaultLocale + ? path.join(docsDir, locale) + : docsDir; if (!existsSync(docsDir)) { throw new Error( `generateDocsSearchFiles found no docs directory at "${docsDir}". Run convertAllMdx first, or check config.outDir.` @@ -197,7 +291,13 @@ export async function generateDocsSearchFiles( } const baseUrl = normalizeBaseUrl(config.baseUrl); - const docs = await readMarkdownDocs(docsDir, baseUrl, config.mounts); + const docs = await readMarkdownDocs( + docsDir, + baseUrl, + config.mounts, + config.i18n, + locale + ); if (docs.length === 0) { throw new Error( `generateDocsSearchFiles found no markdown files under "${docsDir}". Run convertAllMdx first, or check config.outDir.` @@ -211,14 +311,14 @@ export async function generateDocsSearchFiles( } const index = config.embedContent ? indexWithContent : indexWithoutContent; const outputPath = resolveDocsOutputPath( - docsDir, + outputDocsDir, config.outputFile, DEFAULT_OUTPUT_FILE ); const contentOutputPath = config.embedContent ? undefined : resolveDocsOutputPath( - docsDir, + outputDocsDir, config.contentOutputFile, DEFAULT_CONTENT_OUTPUT_FILE ); diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 0c898db..74ed816 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -1,3 +1,4 @@ +import type { LocalizedDocsMetadata } from "../i18n"; import { slugifyDocsHeading } from "../internal/docs-heading"; const DEFAULT_MAX_CHUNK_CHARS = 1200; @@ -40,6 +41,10 @@ const DOCUMENT_DESCRIPTION = 2; const DOCUMENT_URL_PATH = 3; const DOCUMENT_ABSOLUTE_URL = 4; const DOCUMENT_RELATIVE_PATH = 5; +const DOCUMENT_LOCALE = 6; +const DOCUMENT_SOURCE_LOCALE = 7; +const DOCUMENT_IS_FALLBACK = 8; +const DOCUMENT_LOGICAL_PATH = 9; const CHUNK_ID = 0; const CHUNK_DOCUMENT_INDEX = 1; const CHUNK_ANCHOR = 2; @@ -99,7 +104,7 @@ const DEFAULT_SYNONYMS: Record = { typescript: ["ts"], }; -export type DocsSearchDocument = { +export type DocsSearchDocument = LocalizedDocsMetadata & { id?: string; title: string; description?: string; @@ -109,7 +114,7 @@ export type DocsSearchDocument = { content: string; }; -export type DocsSearchDocumentRecord = { +export type DocsSearchDocumentRecord = LocalizedDocsMetadata & { id: string; title: string; description: string; @@ -118,7 +123,7 @@ export type DocsSearchDocumentRecord = { relativePath: string; }; -export type DocsSearchChunk = { +export type DocsSearchChunk = LocalizedDocsMetadata & { id: string; documentId: string; title: string; @@ -142,6 +147,10 @@ export type DocsSearchDocumentEntry = [ urlPath: string, absoluteUrl: string, relativePath: string, + locale?: string, + sourceLocale?: string, + isFallback?: boolean, + logicalPath?: string, ]; export type DocsSearchChunkEntry = [ @@ -198,7 +207,7 @@ export type SearchDocsOptions = ContentStoreOptions & { synonyms?: Record; }; -export type DocsSearchResult = { +export type DocsSearchResult = LocalizedDocsMetadata & { id: string; documentId: string; title: string; @@ -738,7 +747,7 @@ function resolveContentStore( function documentRecordFromEntry( entry: DocsSearchDocumentEntry ): DocsSearchDocumentRecord { - return { + const record: DocsSearchDocumentRecord = { id: entry[DOCUMENT_ID], title: entry[DOCUMENT_TITLE], description: entry[DOCUMENT_DESCRIPTION], @@ -746,6 +755,19 @@ function documentRecordFromEntry( absoluteUrl: entry[DOCUMENT_ABSOLUTE_URL], relativePath: entry[DOCUMENT_RELATIVE_PATH], }; + if (entry[DOCUMENT_LOCALE]) { + record.locale = entry[DOCUMENT_LOCALE]; + } + if (entry[DOCUMENT_SOURCE_LOCALE]) { + record.sourceLocale = entry[DOCUMENT_SOURCE_LOCALE]; + } + if (entry[DOCUMENT_IS_FALLBACK] !== undefined) { + record.isFallback = entry[DOCUMENT_IS_FALLBACK]; + } + if (entry[DOCUMENT_LOGICAL_PATH]) { + record.logicalPath = entry[DOCUMENT_LOGICAL_PATH]; + } + return record; } function chunkFromEntry( @@ -783,6 +805,16 @@ function chunkFromEntry( text, codeText: "", length: entry[CHUNK_LENGTH], + ...(documentRecord.locale ? { locale: documentRecord.locale } : {}), + ...(documentRecord.sourceLocale + ? { sourceLocale: documentRecord.sourceLocale } + : {}), + ...(documentRecord.isFallback === undefined + ? {} + : { isFallback: documentRecord.isFallback }), + ...(documentRecord.logicalPath + ? { logicalPath: documentRecord.logicalPath } + : {}), }; } @@ -815,14 +847,26 @@ export function createDocsSearchIndex( for (const [documentIndex, doc] of markdownDocs.entries()) { const documentId = doc.id ?? `doc-${documentIndex}`; const description = doc.description ?? ""; - documents.push([ + const entry: DocsSearchDocumentEntry = [ documentId, doc.title, description, doc.urlPath, doc.absoluteUrl, doc.relativePath, - ]); + ]; + if ( + doc.locale || + doc.sourceLocale || + doc.isFallback !== undefined || + doc.logicalPath + ) { + entry[DOCUMENT_LOCALE] = doc.locale; + entry[DOCUMENT_SOURCE_LOCALE] = doc.sourceLocale; + entry[DOCUMENT_IS_FALLBACK] = doc.isFallback; + entry[DOCUMENT_LOGICAL_PATH] = doc.logicalPath; + } + documents.push(entry); for (const block of collectSectionBlocks(doc.content)) { const bodyParts = splitWithOverlap( diff --git a/packages/leadtype/src/source/index.ts b/packages/leadtype/src/source/index.ts index 537aa7e..aaf0ca5 100644 --- a/packages/leadtype/src/source/index.ts +++ b/packages/leadtype/src/source/index.ts @@ -22,6 +22,14 @@ import type { Root } from "mdast"; import { glob as fg } from "tinyglobby"; import type { PluggableList } from "unified"; import { convertMdxFile } from "../convert"; +import { + type DocsI18nConfig, + type LocaleCode, + logicalPathFromLocaleRelativePath, + normalizeDocsI18nConfig, + outputRelativePathForLocale, + toLocalizedDocsUrlPath, +} from "../i18n"; import { type DocsPathMount, normalizeBaseUrl, @@ -71,6 +79,10 @@ export type DocsPageMeta = { description: string; /** Group slugs declared in frontmatter. */ groups: string[]; + locale?: LocaleCode; + sourceLocale?: LocaleCode; + isFallback?: boolean; + logicalPath?: string; }; export type DocsPage = DocsPageMeta & { @@ -114,6 +126,9 @@ export type CreateDocsSourceConfig = { toc?: DocsTableOfContentsOptions | false; /** Search-index tuning. */ searchIndex?: CreateDocsSearchIndexOptions; + /** Optional locale configuration. When present, `locale` selects the active docs language. */ + i18n?: DocsI18nConfig; + locale?: LocaleCode; }; export type DocsSource = { @@ -182,33 +197,156 @@ function normalizeGroupValue(value: unknown): string[] { } async function readPageMeta( - filePath: string, - contentDir: string, - mounts?: DocsPathMount[] + selected: SelectedSourceFile, + mounts?: DocsPathMount[], + i18n?: DocsI18nConfig ): Promise { + const { contentDir, filePath } = selected; const relativePath = normalizeDocsPath(path.relative(contentDir, filePath)); const raw = await readFile(filePath, "utf8"); const parsed = parseFrontmatter(raw); const title = String(parsed.data.title ?? "").trim() || - titleFromRelativePath(relativePath); + titleFromRelativePath( + `${selected.logicalPath}${path.extname(relativePath)}` + ); const description = String(parsed.data.description ?? "").trim(); const groups = normalizeGroupValue(parsed.data.group); - const slug = deriveSlug(relativePath); - const urlPath = toDocsUrlPath(relativePath, mounts); const extension = filePath.endsWith(".mdx") ? ".mdx" : ".md"; + const slug = deriveSlug(`${selected.outputRelativePath}${extension}`); + const urlPath = + i18n && selected.locale + ? toLocalizedDocsUrlPath( + `${selected.logicalPath}${extension}`, + selected.locale, + i18n, + mounts + ) + : toDocsUrlPath(relativePath, mounts); return { slug, urlPath, - relativePath: stripDocsExtension(relativePath), + relativePath: selected.outputRelativePath, extension, filePath, title, description, groups, + ...(selected.locale ? { locale: selected.locale } : {}), + ...(selected.sourceLocale ? { sourceLocale: selected.sourceLocale } : {}), + ...(selected.isFallback === undefined + ? {} + : { isFallback: selected.isFallback }), + ...(selected.logicalPath ? { logicalPath: selected.logicalPath } : {}), }; } +type SelectedSourceFile = { + contentDir: string; + filePath: string; + logicalPath: string; + outputRelativePath: string; + locale?: LocaleCode; + sourceLocale?: LocaleCode; + isFallback?: boolean; +}; + +function selectSourceFiles( + files: string[], + contentDir: string, + i18n?: DocsI18nConfig, + locale?: LocaleCode +): SelectedSourceFile[] { + const normalized = normalizeDocsI18nConfig(i18n); + if (!normalized) { + return files.map((filePath) => { + const relativePath = normalizeDocsPath( + path.relative(contentDir, filePath) + ); + return { + contentDir, + filePath, + logicalPath: stripDocsExtension(relativePath), + outputRelativePath: stripDocsExtension(relativePath), + }; + }); + } + + const outputLocale = locale ?? normalized.defaultLocale; + const knownLocale = normalized.locales.some( + (entry) => entry.code === outputLocale + ); + if (!knownLocale) { + throw new Error(`Unknown locale "${outputLocale}" in i18n config.`); + } + + const localeCodes = new Set(normalized.locales.map((entry) => entry.code)); + const relativePaths = files.map((filePath) => + normalizeDocsPath(path.relative(contentDir, filePath)) + ); + const hasRootDefault = relativePaths.some((relativePath) => { + const first = relativePath.split("/")[0] ?? ""; + return !localeCodes.has(first); + }); + const hasDefaultFolder = relativePaths.some((relativePath) => + relativePath.startsWith(`${normalized.defaultLocale}/`) + ); + if (hasRootDefault && hasDefaultFolder) { + throw new Error( + `Ambiguous i18n default-locale layout. Use either root docs files or docs/${normalized.defaultLocale}/ files for the default locale, not both.` + ); + } + + const byLogicalPath = new Map>(); + for (const filePath of files) { + const relativePath = normalizeDocsPath(path.relative(contentDir, filePath)); + const { logicalPath, sourceLocale } = logicalPathFromLocaleRelativePath( + relativePath, + localeCodes + ); + const resolvedSourceLocale = sourceLocale ?? normalized.defaultLocale; + const localeFiles = byLogicalPath.get(logicalPath) ?? new Map(); + const outputRelativePath = outputRelativePathForLocale( + logicalPath, + outputLocale, + i18n + ); + const existing = localeFiles.get(resolvedSourceLocale); + if (existing) { + throw new Error( + `Duplicate docs file for locale "${resolvedSourceLocale}" at "${outputRelativePath}": "${existing.filePath}" conflicts with "${filePath}". Rename one or remove it.` + ); + } + + localeFiles.set(resolvedSourceLocale, { + contentDir, + filePath, + logicalPath, + outputRelativePath, + locale: outputLocale, + sourceLocale: resolvedSourceLocale, + isFallback: resolvedSourceLocale !== outputLocale, + }); + byLogicalPath.set(logicalPath, localeFiles); + } + + const selected: SelectedSourceFile[] = []; + for (const localeFiles of byLogicalPath.values()) { + const direct = localeFiles.get(outputLocale); + const fallback = + outputLocale === normalized.defaultLocale + ? undefined + : localeFiles.get(normalized.defaultLocale); + const match = direct ?? fallback; + if (match) { + selected.push(match); + } + } + return selected.sort((left, right) => + left.outputRelativePath.localeCompare(right.outputRelativePath) + ); +} + export async function createDocsSource( config: CreateDocsSourceConfig ): Promise { @@ -260,8 +398,16 @@ export async function createDocsSource( return cachedMetas; } const files = await listFiles(); + const selectedFiles = selectSourceFiles( + files, + contentDir, + config.i18n, + config.locale + ); const metas = await Promise.all( - files.map((filePath) => readPageMeta(filePath, contentDir, config.mounts)) + selectedFiles.map((file) => + readPageMeta(file, config.mounts, config.i18n) + ) ); // Reject duplicate slugs / urlPaths. Without this guard, two files that // normalize to the same route (e.g. `guide.mdx` and `guide/index.mdx`, or @@ -308,6 +454,8 @@ export async function createDocsSource( baseUrl: config.baseUrl, groups: config.groups ?? [], mounts: config.mounts, + i18n: config.i18n, + locale: config.locale, toc: tocOptions === false ? false : tocOptions, }); } @@ -322,7 +470,23 @@ export async function createDocsSource( const slug = Array.isArray(slugInput) ? slugInput : slugInput.split("/").filter(Boolean); - const meta = await findMetaForSlug(slug); + const normalizedI18n = normalizeDocsI18nConfig(config.i18n); + const localeCodes = new Set( + normalizedI18n?.locales.map((entry) => entry.code) ?? [] + ); + const slugHasLocale = localeCodes.has(slug[0] ?? ""); + let meta = await findMetaForSlug(slug); + if (!meta && slugHasLocale) { + meta = await findMetaForSlug(slug.slice(1)); + } + if ( + !meta && + normalizedI18n && + config.locale && + config.locale !== normalizedI18n.defaultLocale + ) { + meta = await findMetaForSlug([config.locale, ...slug]); + } if (!meta) { return null; } @@ -352,18 +516,26 @@ export async function createDocsSource( async function buildSearchIndex(): Promise { const metas = await listMetas(); const documents: DocsSearchDocument[] = await Promise.all( - metas.map(async (meta) => { - const result = await convertMdxFile(meta.filePath, remarkPlugins); - return { - id: meta.urlPath, - title: meta.title, - description: meta.description, - urlPath: meta.urlPath, - absoluteUrl: toAbsoluteUrl(meta.urlPath, baseUrl), - relativePath: meta.relativePath, - content: result.markdown, - }; - }) + metas + .filter((meta) => !meta.isFallback) + .map(async (meta) => { + const result = await convertMdxFile(meta.filePath, remarkPlugins); + return { + id: meta.urlPath, + title: meta.title, + description: meta.description, + urlPath: meta.urlPath, + absoluteUrl: toAbsoluteUrl(meta.urlPath, baseUrl), + relativePath: meta.relativePath, + ...(meta.locale ? { locale: meta.locale } : {}), + ...(meta.sourceLocale ? { sourceLocale: meta.sourceLocale } : {}), + ...(meta.isFallback === undefined + ? {} + : { isFallback: meta.isFallback }), + ...(meta.logicalPath ? { logicalPath: meta.logicalPath } : {}), + content: result.markdown, + }; + }) ); const index: DocsSearchIndex = createDocsSearchIndex( documents, diff --git a/packages/leadtype/src/source/source.test.ts b/packages/leadtype/src/source/source.test.ts index 07b7e05..157071f 100644 --- a/packages/leadtype/src/source/source.test.ts +++ b/packages/leadtype/src/source/source.test.ts @@ -271,6 +271,91 @@ describe("createDocsSource", () => { expect(documentIds).toEqual(["/docs/guides/setup", "/docs/quickstart"]); }); + it("loads localized pages with default-locale navigation fallback", async () => { + await writeMdx( + path.join(contentDir, "quickstart.mdx"), + "---\ntitle: Quickstart\ngroup: get-started\n---\nEnglish body.\n" + ); + await writeMdx( + path.join(contentDir, "setup.mdx"), + "---\ntitle: Setup\ngroup: get-started\n---\nEnglish setup.\n" + ); + await writeMdx( + path.join(contentDir, "zh/quickstart.mdx"), + "---\ntitle: 快速开始\ngroup: get-started\n---\n中文正文。\n" + ); + + const source = await createDocsSource({ + contentDir, + groups: [{ slug: "get-started", title: "Get Started" }], + i18n: { + defaultLocale: "en", + locales: ["en", "zh"], + }, + locale: "zh", + }); + + const pages = await source.listPages(); + expect( + pages.map((page) => ({ + fallback: page.isFallback, + slug: page.slug.join("/"), + title: page.title, + urlPath: page.urlPath, + })) + ).toEqual([ + { + fallback: false, + slug: "zh/quickstart", + title: "快速开始", + urlPath: "/docs/zh/quickstart", + }, + { + fallback: true, + slug: "zh/setup", + title: "Setup", + urlPath: "/docs/zh/setup", + }, + ]); + + const fallback = await source.loadPage("zh/setup"); + expect(fallback?.isFallback).toBe(true); + expect(fallback?.sourceLocale).toBe("en"); + expect(fallback?.markdown).toContain("English setup"); + + const logicalFallback = await source.loadPage("setup"); + expect(logicalFallback?.urlPath).toBe("/docs/zh/setup"); + + const search = await source.buildSearchIndex(); + expect(search.index.documents.map((entry) => entry[3])).toEqual([ + "/docs/zh/quickstart", + ]); + }); + + it("rejects duplicate localized source files for the same locale and logical path", async () => { + await writeMdx( + path.join(contentDir, "quickstart.md"), + "---\ntitle: Quickstart\n---\nBody.\n" + ); + await writeMdx( + path.join(contentDir, "quickstart.mdx"), + "---\ntitle: Quickstart duplicate\n---\nBody.\n" + ); + + const source = await createDocsSource({ + contentDir, + i18n: { + defaultLocale: "en", + locales: ["en", "zh"], + }, + locale: "en", + }); + + await expect(source.listPages()).rejects.toThrow( + /Duplicate docs file for locale "en"/ + ); + }); + it("getNavigation routes pages into declared groups", async () => { await writeMdx( path.join(contentDir, "guides/setup.mdx"),