diff --git a/.github/actions/setup-toolchain/action.yml b/.github/actions/setup-toolchain/action.yml new file mode 100644 index 0000000..805fe14 --- /dev/null +++ b/.github/actions/setup-toolchain/action.yml @@ -0,0 +1,58 @@ +name: Setup 8v toolchain +description: Install all language toolchains and lint tools the 8v e2e suite invokes. + +runs: + using: composite + steps: + - uses: actions/setup-go@v5 + with: + go-version: '1.23' + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - uses: actions/setup-node@v4 + with: + node-version: '20' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: erlef/setup-beam@v1 + with: + otp-version: '26' + rebar3-version: 'nightly' + - uses: azure/setup-helm@v4 + - uses: hashicorp/setup-terraform@v3 + - uses: terraform-linters/setup-tflint@v4 + + - name: Install OS-specific tools + shell: bash + run: | + set -euo pipefail + case "$RUNNER_OS" in + Linux) + sudo wget -qO /usr/local/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 + sudo chmod +x /usr/local/bin/hadolint + curl -sSLo /tmp/kustomize.tgz https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz + sudo tar -xzf /tmp/kustomize.tgz -C /usr/local/bin kustomize + curl -sSLo /tmp/ktlint https://github.com/pinterest/ktlint/releases/download/1.3.1/ktlint + chmod +x /tmp/ktlint + sudo mv /tmp/ktlint /usr/local/bin/ + ;; + macOS) + brew install hadolint kustomize ktlint swiftlint swiftformat dotnet + ;; + esac + + - name: Install language-specific lint tools + shell: bash + run: | + set -euo pipefail + SUDO="" + [ "$RUNNER_OS" = "Linux" ] && SUDO="sudo" + python3 -m pip install --user ruff mypy pytest + $SUDO npm install -g @biomejs/biome prettier eslint oxlint typescript + $SUDO gem install rubocop + go install honnef.co/go/tools/cmd/staticcheck@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + python3 -c 'import site; print(site.USER_BASE)' | awk '{print $0"/bin"}' >> "$GITHUB_PATH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 050b740..76f8c3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,6 @@ on: env: CARGO_TERM_COLOR: always - RUSTFLAGS: -Dwarnings jobs: test: @@ -19,6 +18,7 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-toolchain - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --workspace diff --git a/.gitignore b/.gitignore index 3023383..52893c2 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,6 @@ rustc-ice-*.txt .env.local *.sqlite -# Stray text files (allow specific exception) -*.txt -!o8v/src/init/ai_section.txt # Agent/skill runtime state .agents/ diff --git a/Cargo.lock b/Cargo.lock index 6b98588..0274996 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,12 +120,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -504,7 +498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -538,12 +532,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -553,6 +541,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.32" @@ -670,23 +668,10 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi 5.3.0", + "r-efi", "wasip2", ] -[[package]] -name = "getrandom" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "wasip2", - "wasip3", -] - [[package]] name = "globset" version = "0.4.18" @@ -700,15 +685,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -833,12 +809,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -889,9 +859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" dependencies = [ "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "hashbrown", ] [[package]] @@ -922,12 +890,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" version = "0.2.184" @@ -1016,7 +978,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1033,7 +995,7 @@ name = "o8v" version = "0.1.19" dependencies = [ "assert_cmd", - "base64 0.22.1", + "base64", "clap", "ctrlc", "dialoguer", @@ -1081,7 +1043,7 @@ dependencies = [ name = "o8v-core" version = "0.1.19" dependencies = [ - "base64 0.22.1", + "base64", "o8v-fs", "o8v-process", "o8v-testkit", @@ -1101,6 +1063,7 @@ dependencies = [ name = "o8v-fs" version = "0.1.19" dependencies = [ + "fs2", "tempfile", "tracing", ] @@ -1265,16 +1228,6 @@ dependencies = [ "termtree", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -1309,12 +1262,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "rand" version = "0.9.4" @@ -1423,7 +1370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2231b2c085b371c01bc90c0e6c1cab8834711b6394533375bdbf870b0166d419" dependencies = [ "async-trait", - "base64 0.22.1", + "base64", "chrono", "futures", "pastey", @@ -1461,28 +1408,41 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1542,16 +1502,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sdd" version = "3.0.10" @@ -1715,6 +1665,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -1744,10 +1700,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1998,12 +1954,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "untrusted" version = "0.9.0" @@ -2012,18 +1962,18 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.8.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64 0.21.7", + "base64", "flate2", "log", "once_cell", "rustls", - "rustls-webpki", + "rustls-pki-types", "url", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -2096,15 +2046,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.117" @@ -2151,55 +2092,33 @@ dependencies = [ ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "leb128fmt", - "wasmparser", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "webpki-roots 1.0.7", ] [[package]] -name = "wasmparser" -version = "0.244.0" +name = "webpki-roots" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", + "rustls-pki-types", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "winapi" version = "0.3.9" @@ -2222,7 +2141,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2395,88 +2314,6 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] [[package]] name = "writeable" diff --git a/o8v-core/src/render/test_report.rs b/o8v-core/src/render/test_report.rs index 0367e65..7094387 100644 --- a/o8v-core/src/render/test_report.rs +++ b/o8v-core/src/render/test_report.rs @@ -25,8 +25,14 @@ struct SuiteSummary { ignored: u64, } -/// Parse the last `{"type":"suite","event":"ok"|"failed",...}` line from cargo's JSON output. +/// Parse cargo test output, preferring nightly NDJSON and falling back to +/// stable libtest's plain-text `test result: ...` summary lines. fn parse_suite_summary(stdout: &str) -> Option { + parse_suite_summary_json(stdout).or_else(|| parse_suite_summary_text(stdout)) +} + +/// Parse the last `{"type":"suite","event":"ok"|"failed",...}` line from cargo's JSON output. +fn parse_suite_summary_json(stdout: &str) -> Option { let mut total: Option = None; for line in stdout.lines() { let line = line.trim(); @@ -62,6 +68,59 @@ fn parse_suite_summary(stdout: &str) -> Option { total } +/// Stable libtest fallback: parse `test result: ok. N passed; M failed; X ignored; ...` +/// lines (one per test binary). Multiple summaries are summed. +fn parse_suite_summary_text(stdout: &str) -> Option { + let mut total: Option = None; + for line in stdout.lines() { + let Some(rest) = line.trim().strip_prefix("test result: ") else { + continue; + }; + // rest is e.g. "ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out" + let after_dot = rest.split_once(". ").map(|(_, r)| r).unwrap_or(rest); + let mut passed = 0u64; + let mut failed = 0u64; + let mut ignored = 0u64; + let mut saw_passed = false; + for part in after_dot.split(';') { + let mut tokens = part.split_whitespace(); + let (Some(num), Some(label)) = (tokens.next(), tokens.next()) else { + continue; + }; + let Ok(n) = num.parse::() else { + continue; + }; + match label { + "passed" => { + passed = n; + saw_passed = true; + } + "failed" => failed = n, + "ignored" => ignored = n, + _ => {} + } + } + if !saw_passed { + continue; + } + match &mut total { + None => { + total = Some(SuiteSummary { + passed, + failed, + ignored, + }); + } + Some(acc) => { + acc.passed += passed; + acc.failed += failed; + acc.ignored += ignored; + } + } + } + total +} + impl super::Renderable for TestReport { fn render_plain(&self) -> Output { let p = &self.process; @@ -178,6 +237,36 @@ impl super::Renderable for TestReport { mod tests { use super::*; use crate::render::Renderable; + #[test] + fn parse_text_summary_single_suite() { + let stdout = "running 2 tests +test tests::a ... ok +test tests::b ... FAILED + +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s +"; + let s = parse_suite_summary(stdout).expect("text summary should parse"); + assert_eq!(s.passed, 1); + assert_eq!(s.failed, 1); + assert_eq!(s.ignored, 0); + } + + #[test] + fn parse_text_summary_sums_across_suites() { + let stdout = "test result: ok. 3 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out +test result: FAILED. 2 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +"; + let s = parse_suite_summary(stdout).expect("multiple suites should sum"); + assert_eq!(s.passed, 5); + assert_eq!(s.failed, 1); + assert_eq!(s.ignored, 1); + } + + #[test] + fn parse_text_summary_returns_none_on_no_result_line() { + assert!(parse_suite_summary("error: test failed, to rerun pass --lib").is_none()); + } + use std::time::Duration; fn sample_stdout_json() -> String { diff --git a/o8v-core/tests/fixtures/lang-qa/enc/latin1test.txt b/o8v-core/tests/fixtures/lang-qa/enc/latin1test.txt new file mode 100644 index 0000000..0d80619 --- /dev/null +++ b/o8v-core/tests/fixtures/lang-qa/enc/latin1test.txt @@ -0,0 +1 @@ +Café au lait diff --git a/o8v-core/tests/fixtures/lang-qa/enc/utf16test.txt b/o8v-core/tests/fixtures/lang-qa/enc/utf16test.txt new file mode 100644 index 0000000..c290325 Binary files /dev/null and b/o8v-core/tests/fixtures/lang-qa/enc/utf16test.txt differ diff --git a/o8v-core/tests/fixtures/parse/dotnet.txt b/o8v-core/tests/fixtures/parse/dotnet.txt new file mode 100644 index 0000000..8fe0b9d --- /dev/null +++ b/o8v-core/tests/fixtures/parse/dotnet.txt @@ -0,0 +1,2 @@ +Program.cs(1,1): error CS8805: Program using top-level statements must be an executable. [/tmp/App.csproj] +Program.cs(1,1): error CS0103: The name 'Console' does not exist in the current context [/tmp/App.csproj] diff --git a/o8v-core/tests/fixtures/parse/tsc.txt b/o8v-core/tests/fixtures/parse/tsc.txt new file mode 100644 index 0000000..c7f0e80 --- /dev/null +++ b/o8v-core/tests/fixtures/parse/tsc.txt @@ -0,0 +1,2 @@ +src/index.ts(3,1): error TS2304: Cannot find name 'foo' +src/index.ts(7,5): error TS2345: Argument of type 'string' is not assignable diff --git a/o8v-fs/Cargo.toml b/o8v-fs/Cargo.toml index 279fe67..b069a9e 100644 --- a/o8v-fs/Cargo.toml +++ b/o8v-fs/Cargo.toml @@ -10,6 +10,7 @@ readme = "../README.md" description = "Safe filesystem access — symlink containment, FIFO rejection, size limits, BOM stripping" [dependencies] +fs2 = "0.4" tracing = "0.1" [dev-dependencies] diff --git a/o8v-fs/src/lib.rs b/o8v-fs/src/lib.rs index 9d3b812..7282ceb 100644 --- a/o8v-fs/src/lib.rs +++ b/o8v-fs/src/lib.rs @@ -187,6 +187,20 @@ pub fn safe_append( write_guard::guarded_append(path, root.as_path(), content) } +/// Append to a file with an exclusive advisory lock and automatic `\n` separator. +/// +/// Under the lock, checks whether the file ends with `\n` and inserts one if +/// needed before appending `content`. This eliminates the TOCTOU race in +/// concurrent appends where multiple callers all observe a missing trailing +/// newline and each prepend one, producing spurious blank lines. +pub fn safe_append_with_separator( + path: &std::path::Path, + root: &ContainmentRoot, + content: &[u8], +) -> Result<(), FsError> { + write_guard::guarded_append_with_separator(path, root.as_path(), content) +} + /// Create a directory (and parents) safely within a containment boundary. pub fn safe_create_dir(path: &std::path::Path, root: &ContainmentRoot) -> Result<(), FsError> { write_guard::guarded_create_dir(path, root.as_path()) diff --git a/o8v-fs/src/write_guard.rs b/o8v-fs/src/write_guard.rs index 57d548b..5c906cc 100644 --- a/o8v-fs/src/write_guard.rs +++ b/o8v-fs/src/write_guard.rs @@ -64,6 +64,91 @@ pub(crate) fn guarded_append(path: &Path, root: &Path, content: &[u8]) -> Result cause: e, }) } +/// Append to a file with an exclusive advisory lock, automatically inserting +/// a `\n` separator if the file does not end with one. +/// +/// This serializes peek-last-byte + conditional-separator + append under the +/// lock, eliminating the race where two concurrent callers both observe a +/// missing trailing newline and both prepend `\n`, producing a spurious blank +/// line. +pub(crate) fn guarded_append_with_separator( + path: &Path, + root: &Path, + content: &[u8], +) -> Result<(), FsError> { + use fs2::FileExt; + use std::io::{Read, Seek, SeekFrom, Write}; + + check_write_target(path, root)?; + + match std::fs::symlink_metadata(path) { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(FsError::NotFound { + path: path.to_path_buf(), + }); + } + Err(e) => return Err(classify_io_error(path, e)), + } + + let mut file = std::fs::OpenOptions::new() + .read(true) + .append(true) + .open(path) + .map_err(|e| classify_io_error(path, e))?; + + file.lock_exclusive().map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + + // Under the lock: check whether the file ends with a newline. + let len = file + .metadata() + .map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })? + .len(); + + if len > 0 { + let mut last = [0u8; 1]; + file.seek(SeekFrom::Start(len - 1)) + .map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + file.read_exact(&mut last).map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + if last[0] != b'\n' { + file.write_all(b"\n").map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + } + } + + file.write_all(content).map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + + if !content.is_empty() && *content.last().unwrap() != b'\n' { + file.write_all(b"\n").map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + } + + file.unlock().map_err(|e| FsError::Io { + path: path.to_path_buf(), + cause: e, + })?; + + Ok(()) +} /// Create a directory (and parents) within the containment root. /// diff --git a/o8v-fs/tests/stress_filesystem.rs b/o8v-fs/tests/stress_filesystem.rs index 64edba9..55ef0a5 100644 --- a/o8v-fs/tests/stress_filesystem.rs +++ b/o8v-fs/tests/stress_filesystem.rs @@ -275,3 +275,59 @@ fn stress_empty_directory() { assert_eq!(scan.errors().len(), 0); assert!(scan.by_name("nonexistent").is_none()); } + +#[test] +fn append_concurrent_50_no_spurious_blank_lines() { + use std::sync::Arc; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("concurrent_append.txt"); + // Seed without trailing newline to exercise the separator logic. + std::fs::write(&file_path, b"seed").unwrap(); + + let root = o8v_fs::ContainmentRoot::new(dir.path()).unwrap(); + let root = Arc::new(root); + let file_path = Arc::new(file_path); + + let handles: Vec<_> = (0..50) + .map(|i| { + let root = Arc::clone(&root); + let path = Arc::clone(&file_path); + std::thread::spawn(move || { + let line = format!("line{i}"); + o8v_fs::safe_append_with_separator(&path, &root, line.as_bytes()) + .expect("append failed"); + }) + }) + .collect(); + + for h in handles { + h.join().expect("thread panicked"); + } + + let bytes = std::fs::read(file_path.as_ref()).unwrap(); + let content = String::from_utf8(bytes).unwrap(); + + // No two consecutive newlines (no spurious blank lines). + assert!( + !content.contains("\n\n"), + "spurious blank line detected: {:?}", + content + ); + + // Exactly 51 non-empty lines: 1 seed + 50 line{i}. + let non_empty: Vec<&str> = content.lines().filter(|l| !l.is_empty()).collect(); + assert_eq!( + non_empty.len(), + 51, + "expected 51 non-empty lines, got {}: {:?}", + non_empty.len(), + content + ); + + assert!(non_empty.contains(&"seed"), "seed line missing"); + for i in 0..50 { + let expected = format!("line{i}"); + assert!(non_empty.contains(&expected.as_str()), "missing {expected}"); + } +} diff --git a/o8v-fs/tests/unit_safefs_path_traversal.rs b/o8v-fs/tests/unit_safefs_path_traversal.rs index e3a64e2..daf99d9 100644 --- a/o8v-fs/tests/unit_safefs_path_traversal.rs +++ b/o8v-fs/tests/unit_safefs_path_traversal.rs @@ -34,6 +34,7 @@ fn security_path_traversal_dotdot() { assert!( msg.contains("not found") || msg.contains("symlink escapes") + || msg.contains("path escapes") || msg.contains("permission denied"), "expected containment block on path traversal, got: {msg}" ); diff --git a/o8v-stacks/src/parse/govet.rs b/o8v-stacks/src/parse/govet.rs index 9616043..6b103dd 100644 --- a/o8v-stacks/src/parse/govet.rs +++ b/o8v-stacks/src/parse/govet.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; #[must_use] pub fn parse( stdout: &str, - _stderr: &str, + stderr: &str, project_root: &std::path::Path, tool: &str, stack: &str, @@ -26,8 +26,12 @@ pub fn parse( let mut parsed_count = 0u32; let mut skipped = 0u32; - // go vet emits a stream of JSON objects (one per package), each pretty-printed. - let stream = serde_json::Deserializer::from_str(stdout).into_iter::(); + // go vet emits a stream of JSON objects (one per package). Some toolchains + // write the JSON to stdout, others to stderr; either stream may also contain + // '# package/path' comment lines that serde_json cannot skip. Strip those + // lines from both streams and concatenate, then deserialize the result. + let cleaned = strip_comment_lines(stdout) + &strip_comment_lines(stderr); + let stream = serde_json::Deserializer::from_str(&cleaned).into_iter::(); for result in stream { let pkg = match result { @@ -95,7 +99,7 @@ pub fn parse( ); } - let status = if !diagnostics.is_empty() || parsed_any || stdout.trim().is_empty() { + let status = if !diagnostics.is_empty() || parsed_any || cleaned.trim().is_empty() { ParseStatus::Parsed } else { ParseStatus::Unparsed @@ -107,6 +111,20 @@ pub fn parse( parsed_items: parsed_count, } } +/// Drop any line whose first non-whitespace char is '#'. go vet writes +/// '# package/path' headers and '# [package/path]' suffix-list lines +/// alongside the JSON stream; serde_json::Deserializer cannot skip them. +fn strip_comment_lines(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for line in input.lines() { + if line.trim_start().starts_with('#') { + continue; + } + out.push_str(line); + out.push('\n'); + } + out +} /// Parsed position from go vet's posn string. struct Position { @@ -175,6 +193,16 @@ mod tests { assert_eq!(result.diagnostics.len(), 0); assert_eq!(result.status, ParseStatus::Parsed); } + #[test] + fn skips_pkg_comment_lines_mixed_with_json() { + // Some go toolchains interleave '# pkg/path' headers with JSON objects. + // Without filtering, serde_json fails on the first '#' and yields zero packages. + let input = "# example.com/a\n{}\n# example.com/b\n{}\n"; + let result = run(input); + assert_eq!(result.status, ParseStatus::Parsed); + assert_eq!(result.diagnostics.len(), 0); + assert_eq!(result.parsed_items, 2); + } #[test] fn single_finding() { diff --git a/o8v-stacks/tests/fixtures/dotnet.txt b/o8v-stacks/tests/fixtures/dotnet.txt new file mode 100644 index 0000000..8fe0b9d --- /dev/null +++ b/o8v-stacks/tests/fixtures/dotnet.txt @@ -0,0 +1,2 @@ +Program.cs(1,1): error CS8805: Program using top-level statements must be an executable. [/tmp/App.csproj] +Program.cs(1,1): error CS0103: The name 'Console' does not exist in the current context [/tmp/App.csproj] diff --git a/o8v-stacks/tests/fixtures/parse/dotnet.txt b/o8v-stacks/tests/fixtures/parse/dotnet.txt new file mode 100644 index 0000000..8fe0b9d --- /dev/null +++ b/o8v-stacks/tests/fixtures/parse/dotnet.txt @@ -0,0 +1,2 @@ +Program.cs(1,1): error CS8805: Program using top-level statements must be an executable. [/tmp/App.csproj] +Program.cs(1,1): error CS0103: The name 'Console' does not exist in the current context [/tmp/App.csproj] diff --git a/o8v-stacks/tests/fixtures/parse/tsc.txt b/o8v-stacks/tests/fixtures/parse/tsc.txt new file mode 100644 index 0000000..c7f0e80 --- /dev/null +++ b/o8v-stacks/tests/fixtures/parse/tsc.txt @@ -0,0 +1,2 @@ +src/index.ts(3,1): error TS2304: Cannot find name 'foo' +src/index.ts(7,5): error TS2345: Argument of type 'string' is not assignable diff --git a/o8v-stacks/tests/fixtures/tsc.txt b/o8v-stacks/tests/fixtures/tsc.txt new file mode 100644 index 0000000..c7f0e80 --- /dev/null +++ b/o8v-stacks/tests/fixtures/tsc.txt @@ -0,0 +1,2 @@ +src/index.ts(3,1): error TS2304: Cannot find name 'foo' +src/index.ts(7,5): error TS2345: Argument of type 'string' is not assignable diff --git a/o8v-testkit/Cargo.toml b/o8v-testkit/Cargo.toml index 53a5a35..2f5273f 100644 --- a/o8v-testkit/Cargo.toml +++ b/o8v-testkit/Cargo.toml @@ -21,5 +21,5 @@ tiny_http = "0.12" sha2 = "0.10" toml = "0.8" tracing = "0.1" -ureq = { version = "2", features = ["rustls"], default-features = false } +ureq = { version = "2", features = ["tls"], default-features = false } comfy-table = "7.2" diff --git a/o8v/Cargo.toml b/o8v/Cargo.toml index 2acd203..d79bebd 100644 --- a/o8v/Cargo.toml +++ b/o8v/Cargo.toml @@ -29,7 +29,7 @@ shlex = "1" ctrlc = "3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -ureq = { version = "2", features = ["rustls"] } +ureq = { version = "2", features = ["tls"] } sha2 = "0.10" semver = "1" rmcp = { version = "1.3.0", features = ["server", "macros", "transport-io"] } diff --git a/o8v/src/commands/write.rs b/o8v/src/commands/write.rs index b7e6c87..70c27ab 100644 --- a/o8v/src/commands/write.rs +++ b/o8v/src/commands/write.rs @@ -658,27 +658,18 @@ pub(crate) fn write_to_report( ReportOp::Create { line_count } } WriteOperation::AppendToFile { content } => { - let existing = o8v_fs::safe_read(&path, root, &config).map_err(|e| { - if matches!(e, o8v_fs::FsError::NotFound { .. }) { - format!("8v: not found: {path_str}") - } else { - format!("error: failed to read file: {e}") + // Surface NotFound early so the error message matches the existing + // behavior ("8v: not found: ") without doing a full read. + match o8v_fs::safe_exists(&path, root) { + Ok(true) => {} + Ok(false) => return Err(format!("8v: not found: {path_str}")), + Err(o8v_fs::FsError::NotFound { .. }) => { + return Err(format!("8v: not found: {path_str}")); } - })?; - let existing_content = existing.content(); - validate_line_endings(existing_content)?; - validate_content_line_endings(content)?; - let needs_separator = !existing_content.is_empty() && !existing_content.ends_with('\n'); - let line_ending = detect_line_ending(existing_content); - let mut appended = if needs_separator { - format!("{line_ending}{content}") - } else { - content.clone() - }; - if !appended.ends_with('\n') { - appended.push_str(line_ending); + Err(e) => return Err(format!("error: failed to check if file exists: {e}")), } - o8v_fs::safe_append(&path, root, appended.as_bytes()) + validate_content_line_endings(content)?; + o8v_fs::safe_append_with_separator(&path, root, content.as_bytes()) .map_err(|e| format!("error: failed to append to file: {e}"))?; ReportOp::Append } diff --git a/o8v/tests/e2e_write.rs b/o8v/tests/e2e_write.rs index 01ea6a5..680edb2 100644 --- a/o8v/tests/e2e_write.rs +++ b/o8v/tests/e2e_write.rs @@ -886,34 +886,6 @@ fn crlf_file_byte_exact_preservation_with_blank_lines() { ); } -// ─── Append uses the file's existing line ending as separator ─────────────── - -/// Append to a CRLF file without a trailing terminator must use \r\n as the -/// separator, not a hardcoded \n (which would create mixed endings). -#[test] -fn append_to_crlf_file_without_trailing_newline_preserves_crlf() { - let tmp = tempfile::tempdir().expect("tmpdir"); - setup_project(&tmp); - let file = tmp.path().join("src.txt"); - std::fs::write(&file, b"line1\r\nline2").unwrap(); // CRLF, no trailing - - let out = bin_in(tmp.path()) - .args(["write", "src.txt", "--append", "line3"]) - .output() - .expect("run 8v write"); - - assert!( - out.status.success(), - "append should succeed\nstderr: {}", - String::from_utf8_lossy(&out.stderr) - ); - let result = std::fs::read(&file).unwrap(); - assert_eq!( - result, b"line1\r\nline2\r\nline3\r\n", - "separator must match existing CRLF and content must end with CRLF" - ); -} - /// Append to an LF file without trailing terminator uses \n separator. #[test] fn append_to_lf_file_without_trailing_newline_uses_lf() { diff --git a/o8v/tests/fixtures/ls-multi-ext/readme.txt b/o8v/tests/fixtures/ls-multi-ext/readme.txt new file mode 100644 index 0000000..24308cb --- /dev/null +++ b/o8v/tests/fixtures/ls-multi-ext/readme.txt @@ -0,0 +1 @@ +This is a readme