diff --git a/.github/ISSUE_TEMPLATE/4-feature-request.yml b/.github/ISSUE_TEMPLATE/4-feature-request.yml new file mode 100644 index 00000000000..fc95a67ec2d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4-feature-request.yml @@ -0,0 +1,31 @@ +name: 🎁 Feature Request +description: Propose a new feature for Codex +labels: + - enhancement + - needs triage +body: + - type: markdown + attributes: + value: | + Is Codex missing a feature that you'd like to see? Feel free to propose it here. + + Before you submit a feature: + 1. Search existing issues for similar features. If you find one, 👍 it rather than opening a new one. + 2. The Codex team will try to balance the varying needs of the community when prioritizing or rejecting new features. Not all features will be accepted. See [Contributing](https://github.com/openai/codex#contributing) for more details. + + - type: textarea + id: feature + attributes: + label: What feature would you like to see? + validations: + required: true + - type: textarea + id: author + attributes: + label: Are you interested in implementing this feature? + description: Please wait for acknowledgement before implementing or opening a PR. + - type: textarea + id: notes + attributes: + label: Additional information + description: Is there anything else you think we should know? diff --git a/.github/actions/codex/bun.lock b/.github/actions/codex/bun.lock index 15b55fafa7f..cd2c52143c4 100644 --- a/.github/actions/codex/bun.lock +++ b/.github/actions/codex/bun.lock @@ -9,7 +9,7 @@ }, "devDependencies": { "@types/bun": "^1.2.20", - "@types/node": "^24.2.1", + "@types/node": "^24.3.0", "prettier": "^3.6.2", "typescript": "^5.9.2", }, @@ -50,7 +50,7 @@ "@types/bun": ["@types/bun@1.2.20", "", { "dependencies": { "bun-types": "1.2.20" } }, "sha512-dX3RGzQ8+KgmMw7CsW4xT5ITBSCrSbfHc36SNT31EOUg/LA9JWq0VDdEXDRSe1InVWpd2yLUM1FUF/kEOyTzYA=="], - "@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@types/node": ["@types/node@24.3.0", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-aPTXCrfwnDLj4VvXrm+UUCQjNEvJgNA8s5F1cvwQU+3KNltTOkBm1j30uNLyqqPNe7gE3KFzImYoZEfLhp4Yow=="], "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], @@ -82,6 +82,8 @@ "@octokit/plugin-rest-endpoint-methods/@octokit/types": ["@octokit/types@12.6.0", "", { "dependencies": { "@octokit/openapi-types": "^20.0.0" } }, "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw=="], + "bun-types/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@octokit/plugin-paginate-rest/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@20.0.0", "", {}, "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="], "@octokit/plugin-rest-endpoint-methods/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@20.0.0", "", {}, "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="], diff --git a/.github/actions/codex/package.json b/.github/actions/codex/package.json index 208d9fef781..8e0e540314c 100644 --- a/.github/actions/codex/package.json +++ b/.github/actions/codex/package.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@types/bun": "^1.2.20", - "@types/node": "^24.2.1", + "@types/node": "^24.3.0", "prettier": "^3.6.2", "typescript": "^5.9.2" } diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index b895d49e20d..bb67fe68961 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -24,3 +24,7 @@ updates: directory: / schedule: interval: weekly + - package-ecosystem: rust-toolchain + directory: codex-rs + schedule: + interval: weekly diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index 82b9eb93b35..f3b85b2b4ba 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -17,6 +17,10 @@ "linux-aarch64": { "regex": "^codex-aarch64-unknown-linux-musl\\.zst$", "path": "codex" + }, + "windows-x86_64": { + "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex.exe" } } } diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..03cedab29c3 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ +# External (non-OpenAI) Pull Request Requirements + +Before opening this Pull Request, please read the "Contributing" section of the README or your PR may be closed: +https://github.com/openai/codex#contributing + +If your PR conforms to our contribution guidelines, replace this text with a detailed and high quality description of your changes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32fb070924d..e62d7bca56f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=4096 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 5737a6bca8b..ce03a430fc5 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Annotate locations with typos uses: codespell-project/codespell-problem-matcher@b80729f885d32f78a716c2f107b4db1025001c42 # v1 - name: Codespell diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml index 6ca0d57f468..58e852336de 100644 --- a/.github/workflows/codex.yml +++ b/.github/workflows/codex.yml @@ -37,9 +37,9 @@ jobs: # Codex is not going to run. - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@1.88 + - uses: dtolnay/rust-toolchain@1.89 with: targets: x86_64-unknown-linux-gnu components: clippy @@ -52,7 +52,7 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ ${{ github.workspace }}/codex-rs/target/ - key: cargo-ubuntu-24.04-x86_64-unknown-linux-gnu-${{ hashFiles('**/Cargo.lock') }} + key: cargo-ubuntu-24.04-x86_64-unknown-linux-gnu-dev-${{ hashFiles('**/Cargo.lock') }} # Note it is possible that the `verify` step internal to Run Codex will # fail, in which case the work to setup the repo was worthless :( diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 0fd67175a90..fa9f1cd15f4 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -1,42 +1,76 @@ name: rust-ci on: - pull_request: - branches: - - main - paths: - - "codex-rs/**" - - ".github/**" + pull_request: {} push: branches: - main - workflow_dispatch: -# For CI, we build in debug (`--profile dev`) rather than release mode so we -# get signal faster. +# CI builds in debug (dev) for faster signal. jobs: - # CI that don't need specific targets + # --- Detect what changed (always runs) ------------------------------------- + changed: + name: Detect changed areas + runs-on: ubuntu-24.04 + outputs: + codex: ${{ steps.detect.outputs.codex }} + workflows: ${{ steps.detect.outputs.workflows }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Detect changed paths (no external action) + id: detect + shell: bash + run: | + set -euo pipefail + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA='${{ github.event.pull_request.base.sha }}' + echo "Base SHA: $BASE_SHA" + # List files changed between base and current HEAD (merge-base aware) + mapfile -t files < <(git diff --name-only --no-renames "$BASE_SHA"...HEAD) + else + # On push / manual runs, default to running everything + files=("codex-rs/force" ".github/force") + fi + + codex=false + workflows=false + for f in "${files[@]}"; do + [[ $f == codex-rs/* ]] && codex=true + [[ $f == .github/* ]] && workflows=true + done + + echo "codex=$codex" >> "$GITHUB_OUTPUT" + echo "workflows=$workflows" >> "$GITHUB_OUTPUT" + + # --- CI that doesn't need specific targets --------------------------------- general: name: Format / etc runs-on: ubuntu-24.04 + needs: changed + if: ${{ needs.changed.outputs.codex == 'true' || needs.changed.outputs.workflows == 'true' || github.event_name == 'push' }} defaults: run: working-directory: codex-rs - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@1.88 + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@1.89 with: components: rustfmt - name: cargo fmt run: cargo fmt -- --config imports_granularity=Item --check - # CI to validate on different os/targets + # --- CI to validate on different os/targets -------------------------------- lint_build_test: name: ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }} runs-on: ${{ matrix.runner }} timeout-minutes: 30 + needs: changed + # Keep job-level if to avoid spinning up runners when not needed + if: ${{ needs.changed.outputs.codex == 'true' || needs.changed.outputs.workflows == 'true' || github.event_name == 'push' }} defaults: run: working-directory: codex-rs @@ -44,8 +78,6 @@ jobs: strategy: fail-fast: false matrix: - # Note: While Codex CLI does not support Windows today, we include - # Windows in CI to ensure the code at least builds there. include: - runner: macos-14 target: aarch64-apple-darwin @@ -53,15 +85,9 @@ jobs: - runner: macos-14 target: x86_64-apple-darwin profile: dev - - runner: macos-14 - target: aarch64-apple-darwin - profile: release - runner: ubuntu-24.04 target: x86_64-unknown-linux-musl profile: dev - - runner: ubuntu-24.04 - target: x86_64-unknown-linux-musl - profile: release - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu profile: dev @@ -75,9 +101,18 @@ jobs: target: x86_64-pc-windows-msvc profile: dev + # Also run representative release builds on Mac and Linux because + # there could be release-only build errors we want to catch. + - runner: macos-14 + target: aarch64-apple-darwin + profile: release + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + profile: release + steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@1.88 + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@1.89 with: targets: ${{ matrix.target }} components: clippy @@ -95,7 +130,7 @@ jobs: - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} name: Install musl build tools run: | - sudo apt install -y musl-tools pkg-config + sudo apt install -y musl-tools pkg-config && sudo rm -rf /var/lib/apt/lists/* - name: cargo clippy id: clippy @@ -104,16 +139,20 @@ jobs: # Running `cargo build` from the workspace root builds the workspace using # the union of all features from third-party crates. This can mask errors # where individual crates have underspecified features. To avoid this, we - # run `cargo build` for each crate individually, though because this is + # run `cargo check` for each crate individually, though because this is # slower, we only do this for the x86_64-unknown-linux-gnu target. - - name: cargo build individual crates - id: build - if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }} + - name: cargo check individual crates + id: cargo_check_all_crates + if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && matrix.profile != 'release' }} continue-on-error: true - run: find . -name Cargo.toml -mindepth 2 -maxdepth 2 -print0 | xargs -0 -n1 -I{} bash -c 'cd "$(dirname "{}")" && cargo build --profile ${{ matrix.profile }}' + run: | + find . -name Cargo.toml -mindepth 2 -maxdepth 2 -print0 \ + | xargs -0 -n1 -I{} bash -c 'cd "$(dirname "{}")" && cargo check --profile ${{ matrix.profile }}' - name: cargo test id: test + # `cargo test` takes too long for release builds to run them on every PR + if: ${{ matrix.profile != 'release' }} continue-on-error: true run: cargo test --all-features --target ${{ matrix.target }} --profile ${{ matrix.profile }} env: @@ -123,8 +162,32 @@ jobs: - name: verify all steps passed if: | steps.clippy.outcome == 'failure' || - steps.build.outcome == 'failure' || + steps.cargo_check_all_crates.outcome == 'failure' || steps.test.outcome == 'failure' run: | - echo "One or more checks failed (clippy, build, or test). See logs for details." + echo "One or more checks failed (clippy, cargo_check_all_crates, or test). See logs for details." exit 1 + + # --- Gatherer job that you mark as the ONLY required status ----------------- + results: + name: CI results (required) + needs: [changed, general, lint_build_test] + if: always() + runs-on: ubuntu-24.04 + steps: + - name: Summarize + shell: bash + run: | + echo "general: ${{ needs.general.result }}" + echo "matrix : ${{ needs.lint_build_test.result }}" + + # If nothing relevant changed (PR touching only root README, etc.), + # declare success regardless of other jobs. + if [[ '${{ needs.changed.outputs.codex }}' != 'true' && '${{ needs.changed.outputs.workflows }}' != 'true' && '${{ github.event_name }}' != 'push' ]]; then + echo 'No relevant changes -> CI not required.' + exit 0 + fi + + # Otherwise require the jobs to have succeeded + [[ '${{ needs.general.result }}' == 'success' ]] || { echo 'general failed'; exit 1; } + [[ '${{ needs.lint_build_test.result }}' == 'success' ]] || { echo 'matrix failed'; exit 1; } diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index d3e313b3ccd..0044b864c78 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -19,7 +19,7 @@ jobs: tag-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Validate tag matches Cargo.toml version shell: bash @@ -74,8 +74,8 @@ jobs: target: x86_64-pc-windows-msvc steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@1.88 + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@1.89 with: targets: ${{ matrix.target }} @@ -117,10 +117,11 @@ jobs: dest="dist/${{ matrix.target }}" # For compatibility with environments that lack the `zstd` tool we - # additionally create a `.tar.gz` alongside every single binary that - # we publish. The end result is: + # additionally create a `.tar.gz` for all platforms and `.zip` for + # Windows alongside every single binary that we publish. The end result is: # codex-.zst (existing) # codex-.tar.gz (new) + # codex-.zip (only for Windows) # 1. Produce a .tar.gz for every file in the directory *before* we # run `zstd --rm`, because that flag deletes the original files. @@ -128,13 +129,20 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.zip ]]; then continue fi # Create per-binary tar.gz tar -C "$dest" -czf "$dest/${base}.tar.gz" "$base" + # Create zip archive for Windows binaries + # Must run from inside the dest dir so 7z won't + # embed the directory path inside the zip. + if [[ "${{ matrix.runner }}" == windows* ]]; then + (cd "$dest" && 7z a "${base}.zip" "$base") + fi + # Also create .zst (existing behaviour) *and* remove the original # uncompressed binary to keep the directory small. zstd -T0 -19 --rm "$dest/$base" @@ -155,7 +163,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - uses: actions/download-artifact@v4 with: diff --git a/.vscode/launch.json b/.vscode/launch.json index 618207f301d..d87ce482e46 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,18 +1,22 @@ { - "version": "0.2.0", - "configurations": [ - { - "type": "lldb", - "request": "launch", - "name": "Cargo launch", - "cargo": { - "cwd": "${workspaceFolder}/codex-rs", - "args": [ - "build", - "--bin=codex-tui" - ] - }, - "args": [] - } - ] + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Cargo launch", + "cargo": { + "cwd": "${workspaceFolder}/codex-rs", + "args": ["build", "--bin=codex-tui"] + }, + "args": [] + }, + { + "type": "lldb", + "request": "attach", + "name": "Attach to running codex CLI", + "pid": "${command:pickProcess}", + "sourceLanguages": ["rust"] + } + ] } diff --git a/AGENTS.md b/AGENTS.md index af254827959..eb2cacd50e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,35 @@ In the codex-rs folder where the rust code lives: - You operate in a sandbox where `CODEX_SANDBOX_NETWORK_DISABLED=1` will be set whenever you use the `shell` tool. Any existing code that uses `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` was authored with this fact in mind. It is often used to early exit out of tests that the author knew you would not be able to run given your sandbox limitations. - Similarly, when you spawn a process using Seatbelt (`/usr/bin/sandbox-exec`), `CODEX_SANDBOX=seatbelt` will be set on the child process. Integration tests that want to run Seatbelt themselves cannot be run under Seatbelt, so checks for `CODEX_SANDBOX=seatbelt` are also often used to early exit out of tests, as appropriate. -Before creating a pull request with changes to `codex-rs`, run `just fmt` (in `codex-rs` directory) to format the code and `just fix` (in `codex-rs` directory) to fix any linter issues in the code, ensure the test suite passes by running `cargo test --all-features` in the `codex-rs` directory. +Before finalizing a change to `codex-rs`, run `just fmt` (in `codex-rs` directory) to format the code and `just fix -p ` (in `codex-rs` directory) to fix any linter issues in the code. Additionally, run the tests: +1. Run the test for the specific project that was changed. For example, if changes were made in `codex-rs/tui`, run `cargo test -p codex-tui`. +2. Once those pass, if any changes were made in common, core, or protocol, run the complete test suite with `cargo test --all-features`. -When making individual changes prefer running tests on individual files or projects first. +## TUI style conventions + +See `codex-rs/tui/styles.md`. + +## TUI code conventions + +- Use concise styling helpers from ratatui’s Stylize trait. + - Basic spans: use "text".into() + - Styled spans: use "text".red(), "text".green(), "text".magenta(), "text".dim(), etc. + - Prefer these over constructing styles with `Span::styled` and `Style` directly. + - Example: patch summary file lines + - Desired: vec![" └ ".into(), "M".red(), " ".dim(), "tui/src/app.rs".dim()] + +## Snapshot tests + +This repo uses snapshot tests (via `insta`), especially in `codex-rs/tui`, to validate rendered output. When UI or text output changes intentionally, update the snapshots as follows: + +- Run tests to generate any updated snapshots: + - `cargo test -p codex-tui` +- Check what’s pending: + - `cargo insta pending-snapshots -p codex-tui` +- Review changes by reading the generated `*.snap.new` files directly in the repo, or preview a specific file: + - `cargo insta show -p codex-tui path/to/file.snap.new` +- Only if you intend to accept all new snapshots in this crate, run: + - `cargo insta accept -p codex-tui` + +If you don’t have the tool: +- `cargo install cargo-insta` diff --git a/README.md b/README.md index 0c01654d664..2ff4c87eb76 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ - [Authenticate locally and copy your credentials to the "headless" machine](#authenticate-locally-and-copy-your-credentials-to-the-headless-machine) - [Connecting through VPS or remote](#connecting-through-vps-or-remote) - [Usage-based billing alternative: Use an OpenAI API key](#usage-based-billing-alternative-use-an-openai-api-key) + - [Forcing a specific auth method (advanced)](#forcing-a-specific-auth-method-advanced) - [Choosing Codex's level of autonomy](#choosing-codexs-level-of-autonomy) - [**1. Read/write**](#1-readwrite) - [**2. Read-only**](#2-read-only) @@ -165,6 +166,35 @@ Notes: - This command only sets the key for your current terminal session, which we recommend. To set it for all future sessions, you can also add the `export` line to your shell's configuration file (e.g., `~/.zshrc`). - If you have signed in with ChatGPT, Codex will default to using your ChatGPT credits. If you wish to use your API key, use the `/logout` command to clear your ChatGPT authentication. +#### Forcing a specific auth method (advanced) + +You can explicitly choose which authentication Codex should prefer when both are available. + +- To always use your API key (even when ChatGPT auth exists), set: + +```toml +# ~/.codex/config.toml +preferred_auth_method = "apikey" +``` + +Or override ad-hoc via CLI: + +```bash +codex --config preferred_auth_method="apikey" +``` + +- To prefer ChatGPT auth (default), set: + +```toml +# ~/.codex/config.toml +preferred_auth_method = "chatgpt" +``` + +Notes: + +- When `preferred_auth_method = "apikey"` and an API key is available, the login screen is skipped. +- When `preferred_auth_method = "chatgpt"` (default), Codex prefers ChatGPT auth if present; if only an API key is present, it will use the API key. Certain account types may also require API-key mode. + ### Choosing Codex's level of autonomy We always recommend running Codex in its default sandbox that gives you strong guardrails around what the agent can do. The default sandbox prevents it from editing files outside its workspace, or from accessing the network. @@ -353,6 +383,13 @@ base_url = "http://my-ollama.example.com:11434/v1" ### Platform sandboxing details +By default, Codex CLI runs code and shell commands inside a restricted sandbox to protect your system. + +> [!IMPORTANT] +> Not all tool calls are sandboxed. Specifically, **trusted Model Context Protocol (MCP) tool calls** are executed outside of the sandbox. +> This is intentional: MCP tools are explicitly configured and trusted by you, and they often need to connect to **external applications or services** (e.g. issue trackers, databases, messaging systems). +> Running them outside the sandbox allows Codex to integrate with these external systems without being blocked by sandbox restrictions. + The mechanism Codex uses to implement the sandbox policy depends on your OS: - **macOS 12+** uses **Apple Seatbelt** and runs commands using `sandbox-exec` with a profile (`-p`) that corresponds to the `--sandbox` that was specified. @@ -566,9 +603,13 @@ We're excited to launch a **$1 million initiative** supporting open source proje ## Contributing -This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! +This project is under active development and the code will likely change pretty significantly. + +**At the moment, we only plan to prioritize reviewing external contributions for bugs or security fixes.** + +If you want to add a new feature or change the behavior of an existing one, please open an issue proposing the feature and get approval from an OpenAI team member before spending time building it. -More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. +**New contributions that don't go through this process may be closed** if they aren't aligned with our current roadmap or conflict with other priorities/upcoming features. ### Development workflow @@ -593,8 +634,9 @@ More broadly we welcome contributions - whether you are opening your very first ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. +2. If your PR adds a new feature that was not previously discussed and approved, we may choose to close your PR (see [Contributing](#contributing)). +3. We may ask for changes - please do not take this personally. We value the work, but we also value consistency and long-term maintainability. +5. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index d92d8f2f4ff..b22c5180c0d 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -43,7 +43,7 @@ switch (platform) { targetTriple = "x86_64-pc-windows-msvc.exe"; break; case "arm64": - // We do not build this today, fall through... + // We do not build this today, fall through... default: break; } @@ -65,9 +65,43 @@ const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); // receives a fatal signal, both processes exit in a predictable manner. const { spawn } = await import("child_process"); +async function tryImport(moduleName) { + try { + // eslint-disable-next-line node/no-unsupported-features/es-syntax + return await import(moduleName); + } catch (err) { + return null; + } +} + +async function resolveRgDir() { + const ripgrep = await tryImport("@vscode/ripgrep"); + if (!ripgrep?.rgPath) { + return null; + } + return path.dirname(ripgrep.rgPath); +} + +function getUpdatedPath(newDirs) { + const pathSep = process.platform === "win32" ? ";" : ":"; + const existingPath = process.env.PATH || ""; + const updatedPath = [ + ...newDirs, + ...existingPath.split(pathSep).filter(Boolean), + ].join(pathSep); + return updatedPath; +} + +const additionalDirs = []; +const rgDir = await resolveRgDir(); +if (rgDir) { + additionalDirs.push(rgDir); +} +const updatedPath = getUpdatedPath(additionalDirs); + const child = spawn(binaryPath, process.argv.slice(2), { stdio: "inherit", - env: { ...process.env, CODEX_MANAGED_BY_NPM: "1" }, + env: { ...process.env, PATH: updatedPath, CODEX_MANAGED_BY_NPM: "1" }, }); child.on("error", (err) => { @@ -120,4 +154,3 @@ if (childResult.type === "signal") { } else { process.exit(childResult.exitCode); } - diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json new file mode 100644 index 00000000000..a1c840ade0e --- /dev/null +++ b/codex-cli/package-lock.json @@ -0,0 +1,119 @@ +{ + "name": "@openai/codex", + "version": "0.0.0-dev", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@openai/codex", + "version": "0.0.0-dev", + "license": "Apache-2.0", + "dependencies": { + "@vscode/ripgrep": "^1.15.14" + }, + "bin": { + "codex": "bin/codex.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@vscode/ripgrep": { + "version": "1.15.14", + "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.14.tgz", + "integrity": "sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "https-proxy-agent": "^7.0.2", + "proxy-from-env": "^1.1.0", + "yauzl": "^2.9.2" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + } + } +} diff --git a/codex-cli/package.json b/codex-cli/package.json index c5464beae54..614ca1a832e 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -16,5 +16,11 @@ "repository": { "type": "git", "url": "git+https://github.com/openai/codex.git" + }, + "dependencies": { + "@vscode/ripgrep": "^1.15.14" + }, + "devDependencies": { + "prettier": "^3.3.3" } } diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 41392633be7..34e79320536 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -176,9 +176,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "arbitrary" @@ -203,6 +203,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -481,6 +487,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-expr" version = "0.15.8" @@ -518,11 +530,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "clap" -version = "4.5.43" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50fd97c9dc2399518aa331917ac6f274280ec5eb34e555dd291899745c48ec6f" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", "clap_derive", @@ -530,9 +548,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.43" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35b5830294e1fa0462034af85cc95225a4cb07092c088c55bda3147cfcd8f65" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -543,18 +561,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.56" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e4efcbb5da11a92e8a609233aa1e8a7d91e38de0be865f016d14700d45a7fd" +checksum = "4d9501bd3f5f09f7bbee01da9a511073ed30a80cd7a509f1214bb74eadea71ad" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", @@ -647,6 +665,8 @@ dependencies = [ "codex-exec", "codex-login", "codex-mcp-server", + "codex-protocol", + "codex-protocol-ts", "codex-tui", "serde_json", "tokio", @@ -660,6 +680,7 @@ version = "0.0.0" dependencies = [ "clap", "codex-core", + "codex-protocol", "serde", "toml 0.9.5", ] @@ -677,11 +698,11 @@ dependencies = [ "codex-apply-patch", "codex-login", "codex-mcp-client", + "codex-protocol", "core_test_support", "dirs", "env-flags", "eventsource-stream", - "fs2", "futures", "landlock", "libc", @@ -716,6 +737,7 @@ dependencies = [ "tree-sitter-bash", "uuid", "walkdir", + "which", "whoami", "wildmatch", "wiremock", @@ -733,6 +755,9 @@ dependencies = [ "codex-common", "codex-core", "codex-ollama", + "codex-protocol", + "core_test_support", + "libc", "owo-colors", "predicates", "serde_json", @@ -741,6 +766,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "wiremock", ] [[package]] @@ -797,13 +823,20 @@ version = "0.0.0" dependencies = [ "base64 0.22.1", "chrono", + "codex-protocol", "pretty_assertions", + "rand 0.8.5", "reqwest", "serde", "serde_json", + "sha2", "tempfile", "thiserror 2.0.12", + "tiny_http", "tokio", + "url", + "urlencoding", + "webbrowser", ] [[package]] @@ -826,7 +859,10 @@ dependencies = [ "anyhow", "assert_cmd", "codex-arg0", + "codex-common", "codex-core", + "codex-login", + "codex-protocol", "mcp-types", "mcp_test_support", "pretty_assertions", @@ -862,11 +898,37 @@ dependencies = [ "wiremock", ] +[[package]] +name = "codex-protocol" +version = "0.0.0" +dependencies = [ + "mcp-types", + "pretty_assertions", + "serde", + "serde_bytes", + "serde_json", + "strum 0.27.2", + "strum_macros 0.27.2", + "ts-rs", + "uuid", +] + +[[package]] +name = "codex-protocol-ts" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "codex-protocol", + "ts-rs", +] + [[package]] name = "codex-tui" version = "0.0.0" dependencies = [ "anyhow", + "async-stream", "base64 0.22.1", "chrono", "clap", @@ -877,6 +939,7 @@ dependencies = [ "codex-file-search", "codex-login", "codex-ollama", + "codex-protocol", "color-eyre", "crossterm", "diffy", @@ -888,7 +951,7 @@ dependencies = [ "once_cell", "path-clean", "pretty_assertions", - "rand 0.8.5", + "rand 0.9.2", "ratatui", "ratatui-image", "regex-lite", @@ -901,6 +964,7 @@ dependencies = [ "supports-color", "textwrap 0.16.2", "tokio", + "tokio-stream", "tracing", "tracing-appender", "tracing-subscriber", @@ -951,6 +1015,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -1005,6 +1079,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1081,6 +1165,7 @@ checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ "bitflags 2.9.1", "crossterm_winapi", + "futures-core", "mio", "parking_lot", "rustix 0.38.44", @@ -1664,16 +1749,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "futures" version = "0.3.31" @@ -2455,6 +2530,28 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.33" @@ -2537,9 +2634,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libfuzzer-sys" @@ -2686,6 +2783,7 @@ version = "0.0.0" dependencies = [ "serde", "serde_json", + "ts-rs", ] [[package]] @@ -2696,8 +2794,10 @@ dependencies = [ "assert_cmd", "codex-core", "codex-mcp-server", + "codex-protocol", "mcp-types", "pretty_assertions", + "serde", "serde_json", "shlex", "tempfile", @@ -2791,6 +2891,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2944,6 +3050,31 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561f357ba7f3a2a61563a186a163d0a3a5247e1089524a3981d49adb775078bc" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900831247d2fe1a09a683278e5384cfb8c80c79fe6b166f9d14bfdde0ea1b03c" +dependencies = [ + "bitflags 2.9.1", + "objc2", +] + [[package]] name = "object" version = "0.36.7" @@ -3670,6 +3801,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -3992,7 +4124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.9.4", "core-foundation-sys", "libc", "security-framework-sys", @@ -4151,6 +4283,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -4515,7 +4658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4572,6 +4715,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.2" @@ -4710,6 +4862,18 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -5059,6 +5223,30 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ts-rs" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef1b7a6d914a34127ed8e1fa927eb7088903787bcded4fa3eef8f85ee1568be" +dependencies = [ + "serde_json", + "thiserror 2.0.12", + "ts-rs-macros", + "uuid", +] + +[[package]] +name = "ts-rs-macros" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ed7b4c18cc150a6a0a1e9ea1ecfa688791220781af6e119f9599a8502a0a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", + "termcolor", +] + [[package]] name = "tui-input" version = "0.14.0" @@ -5162,6 +5350,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5385,12 +5579,40 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf4f3c0ba838e82b4e5ccc4157003fb8c324ee24c058470ffb82820becbde98" +dependencies = [ + "core-foundation 0.10.1", + "jni", + "log", + "ndk-context", + "objc2", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "weezl" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" +[[package]] +name = "which" +version = "6.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +dependencies = [ + "either", + "home", + "rustix 0.38.44", + "winsafe", +] + [[package]] name = "whoami" version = "1.6.0" @@ -5573,6 +5795,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5600,6 +5831,21 @@ dependencies = [ "windows-targets 0.53.2", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5632,6 +5878,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5644,6 +5896,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5656,6 +5914,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5680,6 +5944,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5692,6 +5962,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5704,6 +5980,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5716,6 +5998,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -5737,6 +6025,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + [[package]] name = "wiremock" version = "0.6.4" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 0ed88522284..8a48ef81874 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -15,6 +15,8 @@ members = [ "mcp-server", "mcp-types", "ollama", + "protocol", + "protocol-ts", "tui", ] resolver = "2" diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index 262d219d6db..15966ac29c3 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -22,6 +22,8 @@ use tree_sitter_bash::LANGUAGE as BASH; /// Detailed instructions for gpt-4.1 on how to use the `apply_patch` tool. pub const APPLY_PATCH_TOOL_INSTRUCTIONS: &str = include_str!("../apply_patch_tool_instructions.md"); +const APPLY_PATCH_COMMANDS: [&str; 2] = ["apply_patch", "applypatch"]; + #[derive(Debug, Error, PartialEq)] pub enum ApplyPatchError { #[error(transparent)] @@ -82,7 +84,6 @@ pub struct ApplyPatchArgs { } pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { - const APPLY_PATCH_COMMANDS: [&str; 2] = ["apply_patch", "applypatch"]; match argv { [cmd, body] if APPLY_PATCH_COMMANDS.contains(&cmd.as_str()) => match parse_patch(body) { Ok(source) => MaybeApplyPatch::Body(source), @@ -91,7 +92,9 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { [bash, flag, script] if bash == "bash" && flag == "-lc" - && script.trim_start().starts_with("apply_patch") => + && APPLY_PATCH_COMMANDS + .iter() + .any(|cmd| script.trim_start().starts_with(cmd)) => { match extract_heredoc_body_from_apply_patch_command(script) { Ok(body) => match parse_patch(&body) { @@ -166,7 +169,7 @@ impl ApplyPatchAction { panic!("path must be absolute"); } - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] let filename = path .file_name() .expect("path should not be empty") @@ -179,7 +182,7 @@ impl ApplyPatchAction { *** End Patch"#, ); let changes = HashMap::from([(path.to_path_buf(), ApplyPatchFileChange::Add { content })]); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] Self { changes, cwd: path @@ -262,7 +265,10 @@ pub fn maybe_parse_apply_patch_verified(argv: &[String], cwd: &Path) -> MaybeApp fn extract_heredoc_body_from_apply_patch_command( src: &str, ) -> std::result::Result { - if !src.trim_start().starts_with("apply_patch") { + if !APPLY_PATCH_COMMANDS + .iter() + .any(|cmd| src.trim_start().starts_with(cmd)) + { return Err(ExtractHeredocError::CommandDidNotStartWithApplyPatch); } @@ -415,12 +421,12 @@ fn apply_hunks_to_files(hunks: &[Hunk]) -> anyhow::Result { for hunk in hunks { match hunk { Hunk::AddFile { path, contents } => { - if let Some(parent) = path.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent).with_context(|| { - format!("Failed to create parent directories for {}", path.display()) - })?; - } + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).with_context(|| { + format!("Failed to create parent directories for {}", path.display()) + })?; } std::fs::write(path, contents) .with_context(|| format!("Failed to write file {}", path.display()))?; @@ -439,15 +445,12 @@ fn apply_hunks_to_files(hunks: &[Hunk]) -> anyhow::Result { let AppliedPatch { new_contents, .. } = derive_new_contents_from_chunks(path, chunks)?; if let Some(dest) = move_path { - if let Some(parent) = dest.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent).with_context(|| { - format!( - "Failed to create parent directories for {}", - dest.display() - ) - })?; - } + if let Some(parent) = dest.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).with_context(|| { + format!("Failed to create parent directories for {}", dest.display()) + })?; } std::fs::write(dest, new_contents) .with_context(|| format!("Failed to write file {}", dest.display()))?; @@ -529,9 +532,12 @@ fn compute_replacements( // If a chunk has a `change_context`, we use seek_sequence to find it, then // adjust our `line_index` to continue from there. if let Some(ctx_line) = &chunk.change_context { - if let Some(idx) = - seek_sequence::seek_sequence(original_lines, &[ctx_line.clone()], line_index, false) - { + if let Some(idx) = seek_sequence::seek_sequence( + original_lines, + std::slice::from_ref(ctx_line), + line_index, + false, + ) { line_index = idx + 1; } else { return Err(ApplyPatchError::ComputeReplacements(format!( @@ -682,8 +688,6 @@ pub fn print_summary( #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] - use super::*; use pretty_assertions::assert_eq; use std::fs; @@ -775,6 +779,33 @@ PATCH"#, } } + #[test] + fn test_heredoc_applypatch() { + let args = strs_to_strings(&[ + "bash", + "-lc", + r#"applypatch <<'PATCH' +*** Begin Patch +*** Add File: foo ++hi +*** End Patch +PATCH"#, + ]); + + match maybe_parse_apply_patch(&args) { + MaybeApplyPatch::Body(ApplyPatchArgs { hunks, patch: _ }) => { + assert_eq!( + hunks, + vec![Hunk::AddFile { + path: PathBuf::from("foo"), + contents: "hi\n".to_string() + }] + ); + } + result => panic!("expected MaybeApplyPatch::Body got {result:?}"), + } + } + #[test] fn test_add_file_hunk_creates_file_with_contents() { let dir = tempdir().unwrap(); diff --git a/codex-rs/apply-patch/src/parser.rs b/codex-rs/apply-patch/src/parser.rs index 44c5b146193..ff9dfd6f8fc 100644 --- a/codex-rs/apply-patch/src/parser.rs +++ b/codex-rs/apply-patch/src/parser.rs @@ -427,7 +427,6 @@ fn parse_update_file_chunk( } #[test] -#[allow(clippy::unwrap_used)] fn test_parse_patch() { assert_eq!( parse_patch_text("bad", ParseMode::Strict), diff --git a/codex-rs/arg0/src/lib.rs b/codex-rs/arg0/src/lib.rs index c097ebc11c5..216a0437d13 100644 --- a/codex-rs/arg0/src/lib.rs +++ b/codex-rs/arg0/src/lib.rs @@ -82,10 +82,34 @@ where }) } +const ILLEGAL_ENV_VAR_PREFIX: &str = "CODEX_"; + /// Load env vars from ~/.codex/.env and `$(pwd)/.env`. +/// +/// Security: Do not allow `.env` files to create or modify any variables +/// with names starting with `CODEX_`. fn load_dotenv() { - if let Ok(codex_home) = codex_core::config::find_codex_home() { - dotenvy::from_path(codex_home.join(".env")).ok(); + if let Ok(codex_home) = codex_core::config::find_codex_home() + && let Ok(iter) = dotenvy::from_path_iter(codex_home.join(".env")) + { + set_filtered(iter); + } + + if let Ok(iter) = dotenvy::dotenv_iter() { + set_filtered(iter); + } +} + +/// Helper to set vars from a dotenvy iterator while filtering out `CODEX_` keys. +fn set_filtered(iter: I) +where + I: IntoIterator>, +{ + for (key, value) in iter.into_iter().flatten() { + if !key.to_ascii_uppercase().starts_with(ILLEGAL_ENV_VAR_PREFIX) { + // It is safe to call set_var() because our process is + // single-threaded at this point in its execution. + unsafe { std::env::set_var(&key, &value) }; + } } - dotenvy::dotenv().ok(); } diff --git a/codex-rs/chatgpt/src/chatgpt_token.rs b/codex-rs/chatgpt/src/chatgpt_token.rs index c674afbc570..f003c4392bb 100644 --- a/codex-rs/chatgpt/src/chatgpt_token.rs +++ b/codex-rs/chatgpt/src/chatgpt_token.rs @@ -1,3 +1,4 @@ +use codex_login::AuthMode; use codex_login::CodexAuth; use std::path::Path; use std::sync::LazyLock; @@ -19,7 +20,7 @@ pub fn set_chatgpt_token_data(value: TokenData) { /// Initialize the ChatGPT token from auth.json file pub async fn init_chatgpt_token_from_auth(codex_home: &Path) -> std::io::Result<()> { - let auth = CodexAuth::from_codex_home(codex_home)?; + let auth = CodexAuth::from_codex_home(codex_home, AuthMode::ChatGPT)?; if let Some(auth) = auth { let token_data = auth.get_token_data().await?; set_chatgpt_token_data(token_data); diff --git a/codex-rs/chatgpt/tests/apply_command_e2e.rs b/codex-rs/chatgpt/tests/apply_command_e2e.rs index f1a35e15214..2aa8b809bb7 100644 --- a/codex-rs/chatgpt/tests/apply_command_e2e.rs +++ b/codex-rs/chatgpt/tests/apply_command_e2e.rs @@ -1,5 +1,3 @@ -#![expect(clippy::expect_used)] - use codex_chatgpt::apply_command::apply_diff_from_task; use codex_chatgpt::get_task::GetTaskResponse; use std::path::Path; diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 0f370691cfa..f7af3349e0a 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -25,6 +25,7 @@ codex-core = { path = "../core" } codex-exec = { path = "../exec" } codex-login = { path = "../login" } codex-mcp-server = { path = "../mcp-server" } +codex-protocol = { path = "../protocol" } codex-tui = { path = "../tui" } serde_json = "1" tokio = { version = "1", features = [ @@ -36,3 +37,4 @@ tokio = { version = "1", features = [ ] } tracing = "0.1.41" tracing-subscriber = "0.3.19" +codex-protocol-ts = { path = "../protocol-ts" } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 7f0983cbc6a..6fe7f003c74 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -3,11 +3,11 @@ use std::path::PathBuf; use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; -use codex_core::config_types::SandboxMode; -use codex_core::exec::spawn_command_under_linux_sandbox; use codex_core::exec_env::create_env; +use codex_core::landlock::spawn_command_under_linux_sandbox; use codex_core::seatbelt::spawn_command_under_seatbelt; use codex_core::spawn::StdioPolicy; +use codex_protocol::config_types::SandboxMode; use crate::LandlockCommand; use crate::SeatbeltCommand; diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index 1a70bd27b69..959bf46f182 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -1,20 +1,33 @@ -use std::env; - use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_login::AuthMode; +use codex_login::CLIENT_ID; use codex_login::CodexAuth; use codex_login::OPENAI_API_KEY_ENV_VAR; +use codex_login::ServerOptions; use codex_login::login_with_api_key; -use codex_login::login_with_chatgpt; use codex_login::logout; +use codex_login::run_login_server; +use std::env; +use std::path::PathBuf; + +pub async fn login_with_chatgpt(codex_home: PathBuf) -> std::io::Result<()> { + let opts = ServerOptions::new(codex_home, CLIENT_ID.to_string()); + let server = run_login_server(opts)?; + + eprintln!( + "Starting local login server on http://localhost:{}.\nIf your browser did not open, navigate to this URL to authenticate:\n\n{}", + server.actual_port, server.auth_url, + ); + + server.block_until_done().await +} pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> ! { let config = load_config_or_exit(cli_config_overrides); - let capture_output = false; - match login_with_chatgpt(&config.codex_home, capture_output).await { + match login_with_chatgpt(config.codex_home).await { Ok(_) => { eprintln!("Successfully logged in"); std::process::exit(0); @@ -47,18 +60,18 @@ pub async fn run_login_with_api_key( pub async fn run_login_status(cli_config_overrides: CliConfigOverrides) -> ! { let config = load_config_or_exit(cli_config_overrides); - match CodexAuth::from_codex_home(&config.codex_home) { + match CodexAuth::from_codex_home(&config.codex_home, config.preferred_auth_method) { Ok(Some(auth)) => match auth.mode { AuthMode::ApiKey => match auth.get_token().await { Ok(api_key) => { eprintln!("Logged in using an API key - {}", safe_format_key(&api_key)); - if let Ok(env_api_key) = env::var(OPENAI_API_KEY_ENV_VAR) { - if env_api_key == api_key { - eprintln!( - " API loaded from OPENAI_API_KEY environment variable or .env file" - ); - } + if let Ok(env_api_key) = env::var(OPENAI_API_KEY_ENV_VAR) + && env_api_key == api_key + { + eprintln!( + " API loaded from OPENAI_API_KEY environment variable or .env file" + ); } std::process::exit(0); } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 8f59d2d401a..2acc3d84c50 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -72,6 +72,10 @@ enum Subcommand { /// Apply the latest diff produced by Codex agent as a `git apply` to your local working tree. #[clap(visible_alias = "a")] Apply(ApplyCommand), + + /// Internal: generate TypeScript protocol bindings. + #[clap(hide = true)] + GenerateTs(GenerateTsCommand), } #[derive(Debug, Parser)] @@ -120,6 +124,17 @@ struct LogoutCommand { config_overrides: CliConfigOverrides, } +#[derive(Debug, Parser)] +struct GenerateTsCommand { + /// Output directory where .ts files will be written + #[arg(short = 'o', long = "out", value_name = "DIR")] + out_dir: PathBuf, + + /// Optional path to the Prettier executable to format generated files + #[arg(short = 'p', long = "prettier", value_name = "PRETTIER_BIN")] + prettier: Option, +} + fn main() -> anyhow::Result<()> { arg0_dispatch_or_else(|codex_linux_sandbox_exe| async move { cli_main(codex_linux_sandbox_exe).await?; @@ -144,7 +159,7 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() codex_exec::run_main(exec_cli, codex_linux_sandbox_exe).await?; } Some(Subcommand::Mcp) => { - codex_mcp_server::run_main(codex_linux_sandbox_exe).await?; + codex_mcp_server::run_main(codex_linux_sandbox_exe, cli.config_overrides).await?; } Some(Subcommand::Login(mut login_cli)) => { prepend_config_flags(&mut login_cli.config_overrides, cli.config_overrides); @@ -194,6 +209,9 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() prepend_config_flags(&mut apply_cli.config_overrides, cli.config_overrides); run_apply_command(apply_cli, None).await?; } + Some(Subcommand::GenerateTs(gen_cli)) => { + codex_protocol_ts::generate_ts(&gen_cli.out_dir, gen_cli.prettier.as_deref())?; + } } Ok(()) diff --git a/codex-rs/cli/src/proto.rs b/codex-rs/cli/src/proto.rs index 6c1de7eaa9e..3bc4d81618d 100644 --- a/codex-rs/cli/src/proto.rs +++ b/codex-rs/cli/src/proto.rs @@ -1,15 +1,14 @@ use std::io::IsTerminal; -use std::sync::Arc; use clap::Parser; use codex_common::CliConfigOverrides; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; +use codex_core::NewConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; +use codex_core::protocol::Event; +use codex_core::protocol::EventMsg; use codex_core::protocol::Submission; -use codex_core::util::notify_on_sigint; -use codex_login::CodexAuth; use tokio::io::AsyncBufReadExt; use tokio::io::BufReader; use tracing::error; @@ -36,22 +35,38 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { .map_err(anyhow::Error::msg)?; let config = Config::load_with_cli_overrides(overrides_vec, ConfigOverrides::default())?; - let auth = CodexAuth::from_codex_home(&config.codex_home)?; - let ctrl_c = notify_on_sigint(); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, auth, ctrl_c.clone()).await?; - let codex = Arc::new(codex); + // Use conversation_manager API to start a conversation + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation_id: _, + conversation, + session_configured, + } = conversation_manager.new_conversation(config).await?; + + // Simulate streaming the session_configured event. + let synthetic_event = Event { + // Fake id value. + id: "".to_string(), + msg: EventMsg::SessionConfigured(session_configured), + }; + let session_configured_event = match serde_json::to_string(&synthetic_event) { + Ok(s) => s, + Err(e) => { + error!("Failed to serialize session_configured: {e}"); + return Err(anyhow::Error::from(e)); + } + }; + println!("{session_configured_event}"); // Task that reads JSON lines from stdin and forwards to Submission Queue let sq_fut = { - let codex = codex.clone(); - let ctrl_c = ctrl_c.clone(); + let conversation = conversation.clone(); async move { let stdin = BufReader::new(tokio::io::stdin()); let mut lines = stdin.lines(); loop { let result = tokio::select! { - _ = ctrl_c.notified() => { - info!("Interrupted, exiting"); + _ = tokio::signal::ctrl_c() => { break }, res = lines.next_line() => res, @@ -65,7 +80,7 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { } match serde_json::from_str::(line) { Ok(sub) => { - if let Err(e) = codex.submit_with_id(sub).await { + if let Err(e) = conversation.submit_with_id(sub).await { error!("{e:#}"); break; } @@ -88,8 +103,8 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { let eq_fut = async move { loop { let event = tokio::select! { - _ = ctrl_c.notified() => break, - event = codex.next_event() => event, + _ = tokio::signal::ctrl_c() => break, + event = conversation.next_event() => event, }; match event { Ok(event) => { diff --git a/codex-rs/clippy.toml b/codex-rs/clippy.toml new file mode 100644 index 00000000000..5a6ff7f0523 --- /dev/null +++ b/codex-rs/clippy.toml @@ -0,0 +1,9 @@ +allow-expect-in-tests = true +allow-unwrap-in-tests = true +disallowed-methods = [ + { path = "ratatui::style::Color::Rgb", reason = "Use ANSI colors, which work better in various terminal themes." }, + { path = "ratatui::style::Color::Indexed", reason = "Use ANSI colors, which work better in various terminal themes." }, + { path = "ratatui::style::Stylize::white", reason = "Avoid hardcoding white; prefer default fg or dim/bold. Exception: Disable this rule if rendering over a hardcoded ANSI background." }, + { path = "ratatui::style::Stylize::black", reason = "Avoid hardcoding black; prefer default fg or dim/bold. Exception: Disable this rule if rendering over a hardcoded ANSI background." }, + { path = "ratatui::style::Stylize::yellow", reason = "Avoid yellow; prefer other colors in `tui/styles.md`." }, +] diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index 1723098b8af..b10600574c1 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] clap = { version = "4", features = ["derive", "wrap_help"], optional = true } codex-core = { path = "../core" } +codex-protocol = { path = "../protocol" } serde = { version = "1", optional = true } toml = { version = "0.9", optional = true } diff --git a/codex-rs/common/src/approval_presets.rs b/codex-rs/common/src/approval_presets.rs new file mode 100644 index 00000000000..6dc0cff0eef --- /dev/null +++ b/codex-rs/common/src/approval_presets.rs @@ -0,0 +1,46 @@ +use codex_core::protocol::AskForApproval; +use codex_core::protocol::SandboxPolicy; + +/// A simple preset pairing an approval policy with a sandbox policy. +#[derive(Debug, Clone)] +pub struct ApprovalPreset { + /// Stable identifier for the preset. + pub id: &'static str, + /// Display label shown in UIs. + pub label: &'static str, + /// Short human description shown next to the label in UIs. + pub description: &'static str, + /// Approval policy to apply. + pub approval: AskForApproval, + /// Sandbox policy to apply. + pub sandbox: SandboxPolicy, +} + +/// Built-in list of approval presets that pair approval and sandbox policy. +/// +/// Keep this UI-agnostic so it can be reused by both TUI and MCP server. +pub fn builtin_approval_presets() -> Vec { + vec![ + ApprovalPreset { + id: "read-only", + label: "Read Only", + description: "Codex can read files and answer questions. Codex requires approval to make edits, run commands, or access network", + approval: AskForApproval::OnRequest, + sandbox: SandboxPolicy::ReadOnly, + }, + ApprovalPreset { + id: "auto", + label: "Auto", + description: "Codex can read files, make edits, and run commands in the workspace. Codex requires approval to work outside the workspace or access network", + approval: AskForApproval::OnRequest, + sandbox: SandboxPolicy::new_workspace_write_policy(), + }, + ApprovalPreset { + id: "full-access", + label: "Full Access", + description: "Codex can read files, make edits, and run commands with network access, without approval. Exercise caution", + approval: AskForApproval::Never, + sandbox: SandboxPolicy::DangerFullAccess, + }, + ] +} diff --git a/codex-rs/common/src/config_override.rs b/codex-rs/common/src/config_override.rs index c9b18edc7cf..6b77099524e 100644 --- a/codex-rs/common/src/config_override.rs +++ b/codex-rs/common/src/config_override.rs @@ -142,7 +142,6 @@ fn parse_toml_value(raw: &str) -> Result { } #[cfg(all(test, feature = "cli"))] -#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; diff --git a/codex-rs/common/src/lib.rs b/codex-rs/common/src/lib.rs index 8595262cc09..292503f77e4 100644 --- a/codex-rs/common/src/lib.rs +++ b/codex-rs/common/src/lib.rs @@ -29,3 +29,8 @@ mod config_summary; pub use config_summary::create_config_summary_entries; // Shared fuzzy matcher (used by TUI selection popups and other UI filtering) pub mod fuzzy_match; +// Shared model presets used by TUI and MCP server +pub mod model_presets; +// Shared approval presets (AskForApproval + Sandbox) used by TUI and MCP server +// Not to be confused with AskForApproval, which we should probably rename to EscalationPolicy. +pub mod approval_presets; diff --git a/codex-rs/common/src/model_presets.rs b/codex-rs/common/src/model_presets.rs new file mode 100644 index 00000000000..686a2c0291b --- /dev/null +++ b/codex-rs/common/src/model_presets.rs @@ -0,0 +1,54 @@ +use codex_core::protocol_config_types::ReasoningEffort; + +/// A simple preset pairing a model slug with a reasoning effort. +#[derive(Debug, Clone, Copy)] +pub struct ModelPreset { + /// Stable identifier for the preset. + pub id: &'static str, + /// Display label shown in UIs. + pub label: &'static str, + /// Short human description shown next to the label in UIs. + pub description: &'static str, + /// Model slug (e.g., "gpt-5"). + pub model: &'static str, + /// Reasoning effort to apply for this preset. + pub effort: ReasoningEffort, +} + +/// Built-in list of model presets that pair a model with a reasoning effort. +/// +/// Keep this UI-agnostic so it can be reused by both TUI and MCP server. +pub fn builtin_model_presets() -> &'static [ModelPreset] { + // Order reflects effort from minimal to high. + const PRESETS: &[ModelPreset] = &[ + ModelPreset { + id: "gpt-5-minimal", + label: "gpt-5 minimal", + description: "— fastest responses with limited reasoning; ideal for coding, instructions, or lightweight tasks", + model: "gpt-5", + effort: ReasoningEffort::Minimal, + }, + ModelPreset { + id: "gpt-5-low", + label: "gpt-5 low", + description: "— balances speed with some reasoning; useful for straightforward queries and short explanations", + model: "gpt-5", + effort: ReasoningEffort::Low, + }, + ModelPreset { + id: "gpt-5-medium", + label: "gpt-5 medium", + description: "— default setting; provides a solid balance of reasoning depth and latency for general-purpose tasks", + model: "gpt-5", + effort: ReasoningEffort::Medium, + }, + ModelPreset { + id: "gpt-5-high", + label: "gpt-5 high", + description: "— maximizes reasoning depth for complex or ambiguous problems", + model: "gpt-5", + effort: ReasoningEffort::High, + }, + ]; + PRESETS +} diff --git a/codex-rs/common/src/sandbox_mode_cli_arg.rs b/codex-rs/common/src/sandbox_mode_cli_arg.rs index 588637aebb6..fa5662ce661 100644 --- a/codex-rs/common/src/sandbox_mode_cli_arg.rs +++ b/codex-rs/common/src/sandbox_mode_cli_arg.rs @@ -7,7 +7,7 @@ //! `config.toml`. use clap::ValueEnum; -use codex_core::config_types::SandboxMode; +use codex_protocol::config_types::SandboxMode; #[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] diff --git a/codex-rs/config.md b/codex-rs/config.md index 0d5df17cc80..1af963a16fa 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -149,6 +149,7 @@ approval_policy = "untrusted" ``` If you want to be notified whenever a command fails, use "on-failure": + ```toml # If the command fails when run in the sandbox, Codex asks for permission to # retry the command outside the sandbox. @@ -156,12 +157,14 @@ approval_policy = "on-failure" ``` If you want the model to run until it decides that it needs to ask you for escalated permissions, use "on-request": + ```toml # The model decides when to escalate approval_policy = "on-request" ``` Alternatively, you can have the model run until it is done, and never ask to run a command with escalated permissions: + ```toml # User is never prompted: if the command fails, Codex will automatically try # something out. Note the `exec` subcommand always uses this mode. @@ -217,17 +220,14 @@ Users can specify config values at multiple levels. Order of precedence is as fo ## model_reasoning_effort -If the model name starts with `"o"` (as in `"o3"` or `"o4-mini"`) or `"codex"`, reasoning is enabled by default when using the Responses API. As explained in the [OpenAI Platform documentation](https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning), this can be set to: +If the selected model is known to support reasoning (for example: `o3`, `o4-mini`, `codex-*`, `gpt-5`), reasoning is enabled by default when using the Responses API. As explained in the [OpenAI Platform documentation](https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning), this can be set to: +- `"minimal"` - `"low"` - `"medium"` (default) - `"high"` -To disable reasoning, set `model_reasoning_effort` to `"none"` in your config: - -```toml -model_reasoning_effort = "none" # disable reasoning -``` +Note: to minimize reasoning, choose `"minimal"`. ## model_reasoning_summary @@ -281,6 +281,9 @@ sandbox_mode = "workspace-write" exclude_tmpdir_env_var = false exclude_slash_tmp = false +# Optional list of _additional_ writable roots beyond $TMPDIR and /tmp. +writable_roots = ["/Users/YOU/.pyenv/shims"] + # Allow the command being run inside the sandbox to make outbound network # requests. Disabled by default. network_access = false @@ -297,6 +300,16 @@ This is reasonable to use if Codex is running in an environment that provides it Though using this option may also be necessary if you try to use Codex in environments where its native sandboxing mechanisms are unsupported, such as older Linux kernels or on Windows. +## Approval presets + +Codex provides three main Approval Presets: + +- Read Only: Codex can read files and answer questions; edits, running commands, and network access require approval. +- Auto: Codex can read files, make edits, and run commands in the workspace without approval; asks for approval outside the workspace or for network access. +- Full Access: Full disk and network access without prompts; extremely risky. + +You can further customize how Codex runs at the command line using the `--ask-for-approval` and `--sandbox` options. + ## mcp_servers Defines the list of MCP servers that Codex can consult for tool use. Currently, only servers that are launched by executing a program that communicate over stdio are supported. For servers that use the SSE transport, consider an adapter like [mcp-proxy](https://github.com/sparfenyuk/mcp-proxy). @@ -498,10 +511,12 @@ hide_agent_reasoning = true # defaults to false Surfaces the model’s raw chain-of-thought ("raw reasoning content") when available. Notes: + - Only takes effect if the selected model/provider actually emits raw reasoning content. Many models do not. When unsupported, this option has no visible effect. - Raw reasoning may include intermediate thoughts or sensitive context. Enable only if acceptable for your workflow. Example: + ```toml show_raw_agent_reasoning = true # defaults to false ``` diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index b0b3dc774a1..7e983858a6e 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -19,16 +19,17 @@ chrono = { version = "0.4", features = ["serde"] } codex-apply-patch = { path = "../apply-patch" } codex-login = { path = "../login" } codex-mcp-client = { path = "../mcp-client" } +codex-protocol = { path = "../protocol" } dirs = "6" env-flags = "0.1.1" eventsource-stream = "0.2.3" -fs2 = "0.4.3" futures = "0.3" -libc = "0.2.174" +libc = "0.2.175" mcp-types = { path = "../mcp-types" } mime_guess = "2.0" os_info = "3.12.0" rand = "0.9" +regex-lite = "0.1.6" reqwest = { version = "0.12", features = ["json", "stream"] } regex-lite = "0.1.6" shlex = "1.3.0" @@ -71,13 +72,15 @@ openssl-sys = { version = "*", features = ["vendored"] } [target.aarch64-unknown-linux-musl.dependencies] openssl-sys = { version = "*", features = ["vendored"] } +[target.'cfg(target_os = "windows")'.dependencies] +which = "6" + [dev-dependencies] assert_cmd = "2" core_test_support = { path = "tests/common" } maplit = "1.0.2" predicates = "3" pretty_assertions = "1.4.1" -regex-lite = "0.1.6" tempfile = "3" tokio-test = "0.4" walkdir = "2.5.0" diff --git a/codex-rs/core/src/apply_patch.rs b/codex-rs/core/src/apply_patch.rs index dc11aed0237..4f9292b6d7d 100644 --- a/codex-rs/core/src/apply_patch.rs +++ b/codex-rs/core/src/apply_patch.rs @@ -1,4 +1,5 @@ use crate::codex::Session; +use crate::codex::TurnContext; use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; use crate::protocol::FileChange; @@ -8,7 +9,6 @@ use crate::safety::assess_patch_safety; use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::ApplyPatchFileChange; use std::collections::HashMap; -use std::path::Path; use std::path::PathBuf; pub const CODEX_APPLY_PATCH_ARG1: &str = "--codex-run-as-apply-patch"; @@ -41,21 +41,16 @@ impl From for InternalApplyPatchInvocation { pub(crate) async fn apply_patch( sess: &Session, + turn_context: &TurnContext, sub_id: &str, call_id: &str, action: ApplyPatchAction, ) -> InternalApplyPatchInvocation { - let writable_roots_snapshot = { - #[allow(clippy::unwrap_used)] - let guard = sess.writable_roots.lock().unwrap(); - guard.clone() - }; - match assess_patch_safety( &action, - sess.approval_policy, - &writable_roots_snapshot, - &sess.cwd, + turn_context.approval_policy, + &turn_context.sandbox_policy, + &turn_context.cwd, ) { SafetyCheck::AutoApprove { .. } => { InternalApplyPatchInvocation::DelegateToExec(ApplyPatchExec { @@ -128,30 +123,3 @@ pub(crate) fn convert_apply_patch_to_protocol( } result } - -pub(crate) fn get_writable_roots(cwd: &Path) -> Vec { - let mut writable_roots = Vec::new(); - if cfg!(target_os = "macos") { - // On macOS, $TMPDIR is private to the user. - writable_roots.push(std::env::temp_dir()); - - // Allow pyenv to update its shims directory. Without this, any tool - // that happens to be managed by `pyenv` will fail with an error like: - // - // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable - // - // which is emitted every time `pyenv` tries to run `rehash` (for - // example, after installing a new Python package that drops an entry - // point). Although the sandbox is intentionally read‑only by default, - // writing to the user's local `pyenv` directory is safe because it - // is already user‑writable and scoped to the current user account. - if let Ok(home_dir) = std::env::var("HOME") { - let pyenv_dir = PathBuf::from(home_dir).join(".pyenv"); - writable_roots.push(pyenv_dir); - } - } - - writable_roots.push(cwd.to_path_buf()); - - writable_roots -} diff --git a/codex-rs/core/src/bash.rs b/codex-rs/core/src/bash.rs index b9cd4443566..5b94daf2521 100644 --- a/codex-rs/core/src/bash.rs +++ b/codex-rs/core/src/bash.rs @@ -132,7 +132,6 @@ fn parse_plain_command_from_node(cmd: tree_sitter::Node, src: &str) -> Option Option>> { diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 840e808fc74..f3ed34c6dc9 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -213,7 +213,9 @@ async fn process_chat_sse( let sse = match timeout(idle_timeout, stream.next()).await { Ok(Some(Ok(ev))) => ev, Ok(Some(Err(e))) => { - let _ = tx_event.send(Err(CodexErr::Stream(e.to_string()))).await; + let _ = tx_event + .send(Err(CodexErr::Stream(e.to_string(), None))) + .await; return; } Ok(None) => { @@ -228,7 +230,10 @@ async fn process_chat_sse( } Err(_) => { let _ = tx_event - .send(Err(CodexErr::Stream("idle timeout waiting for SSE".into()))) + .send(Err(CodexErr::Stream( + "idle timeout waiting for SSE".into(), + None, + ))) .await; return; } @@ -285,13 +290,12 @@ async fn process_chat_sse( .get("delta") .and_then(|d| d.get("content")) .and_then(|c| c.as_str()) + && !content.is_empty() { - if !content.is_empty() { - assistant_text.push_str(content); - let _ = tx_event - .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) - .await; - } + assistant_text.push_str(content); + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) + .await; } // Forward any reasoning/thinking deltas if present. @@ -328,27 +332,25 @@ async fn process_chat_sse( .get("delta") .and_then(|d| d.get("tool_calls")) .and_then(|tc| tc.as_array()) + && let Some(tool_call) = tool_calls.first() { - if let Some(tool_call) = tool_calls.first() { - // Mark that we have an active function call in progress. - fn_call_state.active = true; + // Mark that we have an active function call in progress. + fn_call_state.active = true; - // Extract call_id if present. - if let Some(id) = tool_call.get("id").and_then(|v| v.as_str()) { - fn_call_state.call_id.get_or_insert_with(|| id.to_string()); - } + // Extract call_id if present. + if let Some(id) = tool_call.get("id").and_then(|v| v.as_str()) { + fn_call_state.call_id.get_or_insert_with(|| id.to_string()); + } - // Extract function details if present. - if let Some(function) = tool_call.get("function") { - if let Some(name) = function.get("name").and_then(|n| n.as_str()) { - fn_call_state.name.get_or_insert_with(|| name.to_string()); - } + // Extract function details if present. + if let Some(function) = tool_call.get("function") { + if let Some(name) = function.get("name").and_then(|n| n.as_str()) { + fn_call_state.name.get_or_insert_with(|| name.to_string()); + } - if let Some(args_fragment) = - function.get("arguments").and_then(|a| a.as_str()) - { - fn_call_state.arguments.push_str(args_fragment); - } + if let Some(args_fragment) = function.get("arguments").and_then(|a| a.as_str()) + { + fn_call_state.arguments.push_str(args_fragment); } } } @@ -486,15 +488,14 @@ where // Only use the final assistant message if we have not // seen any deltas; otherwise, deltas already built the // cumulative text and this would duplicate it. - if this.cumulative.is_empty() { - if let crate::models::ResponseItem::Message { content, .. } = &item { - if let Some(text) = content.iter().find_map(|c| match c { - crate::models::ContentItem::OutputText { text } => Some(text), - _ => None, - }) { - this.cumulative.push_str(text); - } - } + if this.cumulative.is_empty() + && let crate::models::ResponseItem::Message { content, .. } = &item + && let Some(text) = content.iter().find_map(|c| match c { + crate::models::ContentItem::OutputText { text } => Some(text), + _ => None, + }) + { + this.cumulative.push_str(text); } // Swallow assistant message here; emit on Completed. diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index f0229d45aec..5534e11f36d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -1,5 +1,6 @@ use std::io::BufRead; use std::path::Path; +use std::sync::OnceLock; use std::time::Duration; use bytes::Bytes; @@ -7,6 +8,7 @@ use codex_login::AuthMode; use codex_login::CodexAuth; use eventsource_stream::Eventsource; use futures::prelude::*; +use regex_lite::Regex; use reqwest::StatusCode; use serde::Deserialize; use serde::Serialize; @@ -27,12 +29,11 @@ use crate::client_common::ResponseStream; use crate::client_common::ResponsesApiRequest; use crate::client_common::create_reasoning_param_for_request; use crate::config::Config; -use crate::config_types::ReasoningEffort as ReasoningEffortConfig; -use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::CodexErr; use crate::error::Result; use crate::error::UsageLimitReachedError; use crate::flags::CODEX_RS_SSE_FIXTURE; +use crate::model_family::ModelFamily; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::models::ResponseItem; @@ -40,6 +41,8 @@ use crate::openai_tools::create_tools_json_for_responses_api; use crate::protocol::TokenUsage; use crate::user_agent::get_codex_user_agent; use crate::util::backoff; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use std::sync::Arc; #[derive(Debug, Deserialize)] @@ -49,10 +52,12 @@ struct ErrorResponse { #[derive(Debug, Deserialize)] struct Error { - r#type: String, + r#type: Option, + code: Option, + message: Option, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct ModelClient { config: Arc, auth: Option, @@ -203,11 +208,7 @@ impl ModelClient { req_builder = req_builder.header("chatgpt-account-id", account_id); } - let originator = self - .config - .internal_originator - .as_deref() - .unwrap_or("codex_cli_rs"); + let originator = &self.config.responses_originator_header; req_builder = req_builder.header("originator", originator); req_builder = req_builder.header("User-Agent", get_codex_user_agent(Some(originator))); @@ -247,6 +248,12 @@ impl ModelClient { .and_then(|v| v.to_str().ok()) .and_then(|s| s.parse::().ok()); + if status == StatusCode::UNAUTHORIZED + && let Some(a) = auth.as_ref() + { + let _ = a.refresh_token().await; + } + // The OpenAI Responses endpoint returns structured JSON bodies even for 4xx/5xx // errors. When we bubble early with only the HTTP status the caller sees an opaque // "unexpected status 400 Bad Request" which makes debugging nearly impossible. @@ -254,7 +261,10 @@ impl ModelClient { // exact error message (e.g. "Unknown parameter: 'input[0].metadata'"). The body is // small and this branch only runs on error paths so the extra allocation is // negligible. - if !(status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) { + if !(status == StatusCode::TOO_MANY_REQUESTS + || status == StatusCode::UNAUTHORIZED + || status.is_server_error()) + { // Surface the error body to callers. Use `unwrap_or_default` per Clippy. let body = res.text().await.unwrap_or_default(); return Err(CodexErr::UnexpectedStatus(status, body)); @@ -263,14 +273,18 @@ impl ModelClient { if status == StatusCode::TOO_MANY_REQUESTS { let body = res.json::().await.ok(); if let Some(ErrorResponse { - error: Error { r#type, .. }, + error: + Error { + r#type: Some(error_type), + .. + }, }) = body { - if r#type == "usage_limit_reached" { + if error_type == "usage_limit_reached" { return Err(CodexErr::UsageLimitReached(UsageLimitReachedError { plan_type: auth.and_then(|a| a.get_plan_type()), })); - } else if r#type == "usage_not_included" { + } else if error_type == "usage_not_included" { return Err(CodexErr::UsageNotIncluded); } } @@ -303,6 +317,30 @@ impl ModelClient { pub fn get_provider(&self) -> ModelProviderInfo { self.provider.clone() } + + /// Returns the currently configured model slug. + pub fn get_model(&self) -> String { + self.config.model.clone() + } + + /// Returns the currently configured model family. + pub fn get_model_family(&self) -> ModelFamily { + self.config.model_family.clone() + } + + /// Returns the current reasoning effort setting. + pub fn get_reasoning_effort(&self) -> ReasoningEffortConfig { + self.effort + } + + /// Returns the current reasoning summary setting. + pub fn get_reasoning_summary(&self) -> ReasoningSummaryConfig { + self.summary + } + + pub fn get_auth(&self) -> Option { + self.auth.clone() + } } #[derive(Debug, Deserialize, Serialize)] @@ -366,13 +404,14 @@ async fn process_sse( // If the stream stays completely silent for an extended period treat it as disconnected. // The response id returned from the "complete" message. let mut response_completed: Option = None; + let mut response_error: Option = None; loop { let sse = match timeout(idle_timeout, stream.next()).await { Ok(Some(Ok(sse))) => sse, Ok(Some(Err(e))) => { debug!("SSE Error: {e:#}"); - let event = CodexErr::Stream(e.to_string()); + let event = CodexErr::Stream(e.to_string(), None); let _ = tx_event.send(Err(event)).await; return; } @@ -390,9 +429,10 @@ async fn process_sse( } None => { let _ = tx_event - .send(Err(CodexErr::Stream( + .send(Err(response_error.unwrap_or(CodexErr::Stream( "stream closed before response.completed".into(), - ))) + None, + )))) .await; } } @@ -400,7 +440,10 @@ async fn process_sse( } Err(_) => { let _ = tx_event - .send(Err(CodexErr::Stream("idle timeout waiting for SSE".into()))) + .send(Err(CodexErr::Stream( + "idle timeout waiting for SSE".into(), + None, + ))) .await; return; } @@ -478,15 +521,25 @@ async fn process_sse( } "response.failed" => { if let Some(resp_val) = event.response { - let error = resp_val - .get("error") - .and_then(|v| v.get("message")) - .and_then(|v| v.as_str()) - .unwrap_or("response.failed event received"); - - let _ = tx_event - .send(Err(CodexErr::Stream(error.to_string()))) - .await; + response_error = Some(CodexErr::Stream( + "response.failed event received".to_string(), + None, + )); + + let error = resp_val.get("error"); + + if let Some(error) = error { + match serde_json::from_value::(error.clone()) { + Ok(error) => { + let delay = try_parse_retry_after(&error); + let message = error.message.unwrap_or_default(); + response_error = Some(CodexErr::Stream(message, delay)); + } + Err(e) => { + debug!("failed to parse ErrorResponse: {e}"); + } + } + } } } // Final response completed – includes array of output items & id @@ -550,10 +603,42 @@ async fn stream_from_fixture( Ok(ResponseStream { rx_event }) } +fn rate_limit_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + + #[expect(clippy::unwrap_used)] + RE.get_or_init(|| Regex::new(r"Please try again in (\d+(?:\.\d+)?)(s|ms)").unwrap()) +} + +fn try_parse_retry_after(err: &Error) -> Option { + if err.code != Some("rate_limit_exceeded".to_string()) { + return None; + } + + // parse the Please try again in 1.898s format using regex + let re = rate_limit_regex(); + if let Some(message) = &err.message + && let Some(captures) = re.captures(message) + { + let seconds = captures.get(1); + let unit = captures.get(2); + + if let (Some(value), Some(unit)) = (seconds, unit) { + let value = value.as_str().parse::().ok()?; + let unit = unit.as_str(); + + if unit == "s" { + return Some(Duration::from_secs_f64(value)); + } else if unit == "ms" { + return Some(Duration::from_millis(value as u64)); + } + } + } + None +} + #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] - use super::*; use serde_json::json; use tokio::sync::mpsc; @@ -735,13 +820,49 @@ mod tests { matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); match &events[1] { - Err(CodexErr::Stream(msg)) => { + Err(CodexErr::Stream(msg, _)) => { assert_eq!(msg, "stream closed before response.completed") } other => panic!("unexpected second event: {other:?}"), } } + #[tokio::test] + async fn error_when_error_event() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + let provider = ModelProviderInfo { + name: "test".to_string(), + base_url: Some("https://test.com".to_string()), + env_key: Some("TEST_API_KEY".to_string()), + env_key_instructions: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: Some(0), + stream_max_retries: Some(0), + stream_idle_timeout_ms: Some(1000), + requires_openai_auth: false, + }; + + let events = collect_events(&[sse1.as_bytes()], provider).await; + + assert_eq!(events.len(), 1); + + match &events[0] { + Err(CodexErr::Stream(msg, delay)) => { + assert_eq!( + msg, + "Rate limit reached for gpt-5 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." + ); + assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); + } + other => panic!("unexpected second event: {other:?}"), + } + } + // ──────────────────────────── // Table-driven test from `main` // ──────────────────────────── @@ -840,4 +961,27 @@ mod tests { ); } } + + #[test] + fn test_try_parse_retry_after() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + }; + + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_millis(28))); + } + + #[test] + fn test_try_parse_retry_after_no_delay() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); + } } diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 440d250b625..15b8ea89c0b 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,19 +1,15 @@ -use crate::config_types::ReasoningEffort as ReasoningEffortConfig; -use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; use crate::model_family::ModelFamily; use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::OpenAiTool; -use crate::protocol::AskForApproval; -use crate::protocol::SandboxPolicy; use crate::protocol::TokenUsage; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use futures::Stream; use serde::Serialize; use std::borrow::Cow; -use std::fmt::Display; -use std::path::PathBuf; use std::pin::Pin; use std::task::Context; use std::task::Poll; @@ -23,62 +19,19 @@ use tokio::sync::mpsc; /// with this content. const BASE_INSTRUCTIONS: &str = include_str!("../prompt.md"); -/// wraps environment context message in a tag for the model to parse more easily. -const ENVIRONMENT_CONTEXT_START: &str = "\n\n"; -const ENVIRONMENT_CONTEXT_END: &str = "\n\n"; - /// wraps user instructions message in a tag for the model to parse more easily. const USER_INSTRUCTIONS_START: &str = "\n\n"; const USER_INSTRUCTIONS_END: &str = "\n\n"; -#[derive(Debug, Clone)] -pub(crate) struct EnvironmentContext { - pub cwd: PathBuf, - pub approval_policy: AskForApproval, - pub sandbox_policy: SandboxPolicy, -} - -impl Display for EnvironmentContext { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "Current working directory: {}", - self.cwd.to_string_lossy() - )?; - writeln!(f, "Approval policy: {}", self.approval_policy)?; - writeln!(f, "Sandbox policy: {}", self.sandbox_policy)?; - - let network_access = match self.sandbox_policy.clone() { - SandboxPolicy::DangerFullAccess => "enabled", - SandboxPolicy::ReadOnly => "restricted", - SandboxPolicy::WorkspaceWrite { network_access, .. } => { - if network_access { - "enabled" - } else { - "restricted" - } - } - }; - writeln!(f, "Network access: {network_access}")?; - Ok(()) - } -} - -/// API request payload for a single model turn. +/// API request payload for a single model turn #[derive(Default, Debug, Clone)] pub struct Prompt { /// Conversation context input items. pub input: Vec, - /// Optional instructions from the user to amend to the built-in agent - /// instructions. - pub user_instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). pub store: bool, - /// A list of key-value pairs that will be added as a developer message - /// for the model to use - pub environment_context: Option, - /// Tools available to the model, including additional tools sourced from /// external MCP servers. pub tools: Vec, @@ -100,36 +53,19 @@ impl Prompt { Cow::Owned(sections.join("\n")) } - fn get_formatted_user_instructions(&self) -> Option { - self.user_instructions - .as_ref() - .map(|ui| format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}")) - } - - fn get_formatted_environment_context(&self) -> Option { - self.environment_context - .as_ref() - .map(|ec| format!("{ENVIRONMENT_CONTEXT_START}{ec}{ENVIRONMENT_CONTEXT_END}")) + pub(crate) fn get_formatted_input(&self) -> Vec { + self.input.clone() } - pub(crate) fn get_formatted_input(&self) -> Vec { - let mut input_with_instructions = Vec::with_capacity(self.input.len() + 2); - if let Some(ec) = self.get_formatted_environment_context() { - input_with_instructions.push(ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: ec }], - }); + /// Creates a formatted user instructions message from a string + pub(crate) fn format_user_instructions_message(ui: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}"), + }], } - if let Some(ui) = self.get_formatted_user_instructions() { - input_with_instructions.push(ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: ui }], - }); - } - input_with_instructions.extend(self.input.clone()); - input_with_instructions } } @@ -149,53 +85,8 @@ pub enum ResponseEvent { #[derive(Debug, Serialize)] pub(crate) struct Reasoning { - pub(crate) effort: OpenAiReasoningEffort, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) summary: Option, -} - -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning -#[derive(Debug, Serialize, Default, Clone, Copy)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiReasoningEffort { - Low, - #[default] - Medium, - High, -} - -impl From for Option { - fn from(effort: ReasoningEffortConfig) -> Self { - match effort { - ReasoningEffortConfig::Low => Some(OpenAiReasoningEffort::Low), - ReasoningEffortConfig::Medium => Some(OpenAiReasoningEffort::Medium), - ReasoningEffortConfig::High => Some(OpenAiReasoningEffort::High), - ReasoningEffortConfig::None => None, - } - } -} - -/// A summary of the reasoning performed by the model. This can be useful for -/// debugging and understanding the model's reasoning process. -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries -#[derive(Debug, Serialize, Default, Clone, Copy)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiReasoningSummary { - #[default] - Auto, - Concise, - Detailed, -} - -impl From for Option { - fn from(summary: ReasoningSummaryConfig) -> Self { - match summary { - ReasoningSummaryConfig::Auto => Some(OpenAiReasoningSummary::Auto), - ReasoningSummaryConfig::Concise => Some(OpenAiReasoningSummary::Concise), - ReasoningSummaryConfig::Detailed => Some(OpenAiReasoningSummary::Detailed), - ReasoningSummaryConfig::None => None, - } - } + pub(crate) effort: ReasoningEffortConfig, + pub(crate) summary: ReasoningSummaryConfig, } /// Request object that is serialized as JSON and POST'ed when using the @@ -226,12 +117,7 @@ pub(crate) fn create_reasoning_param_for_request( summary: ReasoningSummaryConfig, ) -> Option { if model_family.supports_reasoning_summaries { - let effort: Option = effort.into(); - let effort = effort?; - Some(Reasoning { - effort, - summary: summary.into(), - }) + Some(Reasoning { effort, summary }) } else { None } @@ -251,7 +137,6 @@ impl Stream for ResponseStream { #[cfg(test)] mod tests { - #![allow(clippy::expect_used)] use crate::model_family::find_family_for_model; use super::*; @@ -259,7 +144,6 @@ mod tests { #[test] fn get_full_instructions_no_user_content() { let prompt = Prompt { - user_instructions: Some("custom instruction".to_string()), ..Default::default() }; let expected = format!("{BASE_INSTRUCTIONS}\n{APPLY_PATCH_TOOL_INSTRUCTIONS}"); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5e64d8d815b..7d616f96e0b 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1,12 +1,10 @@ -// Poisoned mutex should fail the program -#![allow(clippy::unwrap_used)] - use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use std::sync::MutexGuard; use std::sync::atomic::AtomicU64; use std::time::Duration; @@ -16,11 +14,12 @@ use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::MaybeApplyPatchVerified; use codex_apply_patch::maybe_parse_apply_patch_verified; use codex_login::CodexAuth; +use codex_protocol::protocol::TurnAbortReason; +use codex_protocol::protocol::TurnAbortedEvent; use futures::prelude::*; use mcp_types::CallToolResult; use serde::Serialize; use serde_json; -use tokio::sync::Notify; use tokio::sync::oneshot; use tokio::task::AbortHandle; use tracing::debug; @@ -30,19 +29,19 @@ use tracing::trace; use tracing::warn; use uuid::Uuid; +use crate::ModelProviderInfo; +use crate::apply_patch; use crate::apply_patch::ApplyPatchExec; use crate::apply_patch::CODEX_APPLY_PATCH_ARG1; use crate::apply_patch::InternalApplyPatchInvocation; use crate::apply_patch::convert_apply_patch_to_protocol; -use crate::apply_patch::get_writable_roots; -use crate::apply_patch::{self}; use crate::client::ModelClient; -use crate::client_common::EnvironmentContext; use crate::client_common::Prompt; use crate::client_common::ResponseEvent; use crate::config::Config; use crate::config_types::ShellEnvironmentPolicy; use crate::conversation_history::ConversationHistory; +use crate::environment_context::EnvironmentContext; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::error::SandboxErr; @@ -56,6 +55,7 @@ use crate::exec::process_exec_tool_call; use crate::exec_env::create_env; use crate::mcp_connection_manager::McpConnectionManager; use crate::mcp_tool_call::handle_mcp_tool_call; +use crate::model_family::find_family_for_model; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; use crate::models::LocalShellAction; @@ -64,6 +64,7 @@ use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::models::ShellToolCallParams; +use crate::openai_tools::ApplyPatchToolArgs; use crate::openai_tools::ToolsConfig; use crate::openai_tools::get_openai_tools; use crate::parse_command::parse_command; @@ -93,6 +94,7 @@ use crate::protocol::PatchApplyEndEvent; use crate::protocol::ReviewDecision; use crate::protocol::SandboxPolicy; use crate::protocol::SessionConfiguredEvent; +use crate::protocol::StreamErrorEvent; use crate::protocol::Submission; use crate::protocol::TaskCompleteEvent; use crate::protocol::TurnDiffEvent; @@ -104,7 +106,23 @@ use crate::shell; use crate::turn_diff_tracker::TurnDiffTracker; use crate::user_notification::UserNotification; use crate::util::backoff; -use regex_lite::Regex; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; + +// A convenience extension trait for acquiring mutex locks where poisoning is +// unrecoverable and should abort the program. This avoids scattered `.unwrap()` +// calls on `lock()` while still surfacing a clear panic message when a lock is +// poisoned. +trait MutexExt { + fn lock_unchecked(&self) -> MutexGuard<'_, T>; +} + +impl MutexExt for Mutex { + fn lock_unchecked(&self) -> MutexGuard<'_, T> { + #[expect(clippy::expect_used)] + self.lock().expect("poisoned lock") + } +} /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -119,26 +137,23 @@ pub struct Codex { /// unique session id. pub struct CodexSpawnOk { pub codex: Codex, - pub init_id: String, pub session_id: Uuid, } +pub(crate) const INITIAL_SUBMIT_ID: &str = ""; + impl Codex { /// Spawn a new [`Codex`] and initialize the session. - pub async fn spawn( - config: Config, - auth: Option, - ctrl_c: Arc, - ) -> CodexResult { - // experimental resume path (undocumented) - let resume_path = config.experimental_resume.clone(); - info!("resume_path: {resume_path:?}"); + pub async fn spawn(config: Config, auth: Option) -> CodexResult { let (tx_sub, rx_sub) = async_channel::bounded(64); let (tx_event, rx_event) = async_channel::unbounded(); let user_instructions = get_user_instructions(&config).await; - let configure_session = Op::ConfigureSession { + let config = Arc::new(config); + let resume_path = config.experimental_resume.clone(); + + let configure_session = ConfigureSession { provider: config.model_provider.clone(), model: config.model.clone(), model_reasoning_effort: config.model_reasoning_effort, @@ -150,28 +165,28 @@ impl Codex { disable_response_storage: config.disable_response_storage, notify: config.notify.clone(), cwd: config.cwd.clone(), - resume_path: resume_path.clone(), + resume_path, }; - let config = Arc::new(config); - // Generate a unique ID for the lifetime of this Codex session. - let session_id = Uuid::new_v4(); - tokio::spawn(submission_loop( - session_id, config, auth, rx_sub, tx_event, ctrl_c, - )); + let (session, turn_context) = + Session::new(configure_session, config.clone(), auth, tx_event.clone()) + .await + .map_err(|e| { + error!("Failed to create session: {e:#}"); + CodexErr::InternalAgentDied + })?; + let session_id = session.session_id; + + // This task will run until Op::Shutdown is received. + tokio::spawn(submission_loop(session, turn_context, config, rx_sub)); let codex = Codex { next_id: AtomicU64::new(0), tx_sub, rx_event, }; - let init_id = codex.submit(configure_session).await?; - Ok(CodexSpawnOk { - codex, - init_id, - session_id, - }) + Ok(CodexSpawnOk { codex, session_id }) } /// Submit the `op` wrapped in a `Submission` with a unique ID. @@ -205,26 +220,22 @@ impl Codex { } } +/// Mutable state of the agent +#[derive(Default)] +struct State { + approved_commands: HashSet>, + current_task: Option, + pending_approvals: HashMap>, + pending_input: Vec, + history: ConversationHistory, +} + /// Context for an initialized model agent /// /// A session has at most 1 running task at a time, and can be interrupted by user input. pub(crate) struct Session { - client: ModelClient, - pub(crate) tx_event: Sender, - ctrl_c: Arc, - - /// The session's current working directory. All relative paths provided by - /// the model as well as sandbox policies are resolved against this path - /// instead of `std::env::current_dir()`. - pub(crate) cwd: PathBuf, - base_instructions: Option, - user_instructions: Option, - pub(crate) approval_policy: AskForApproval, - sandbox_policy: SandboxPolicy, - shell_environment_policy: ShellEnvironmentPolicy, - pub(crate) writable_roots: Mutex>, - disable_response_storage: bool, - tools_config: ToolsConfig, + session_id: Uuid, + tx_event: Sender, /// Manager for external MCP servers/tools. mcp_connection_manager: McpConnectionManager, @@ -242,7 +253,24 @@ pub(crate) struct Session { show_raw_agent_reasoning: bool, } -impl Session { +/// The context needed for a single turn of the conversation. +#[derive(Debug)] +pub(crate) struct TurnContext { + pub(crate) client: ModelClient, + /// The session's current working directory. All relative paths provided by + /// the model as well as sandbox policies are resolved against this path + /// instead of `std::env::current_dir()`. + pub(crate) cwd: PathBuf, + pub(crate) base_instructions: Option, + pub(crate) user_instructions: Option, + pub(crate) approval_policy: AskForApproval, + pub(crate) sandbox_policy: SandboxPolicy, + pub(crate) shell_environment_policy: ShellEnvironmentPolicy, + pub(crate) disable_response_storage: bool, + pub(crate) tools_config: ToolsConfig, +} + +impl TurnContext { fn resolve_path(&self, path: Option) -> PathBuf { path.as_ref() .map(PathBuf::from) @@ -250,250 +278,278 @@ impl Session { } } -/// Mutable state of the agent -#[derive(Default)] -struct State { - approved_commands: HashSet>, - current_task: Option, - pending_approvals: HashMap>, - pending_input: Vec, - history: ConversationHistory, -} - -/// Heuristically detect image file paths inside user text and attach them as -/// `InputItem::LocalImage`, preserving the original text. This makes mid-chat -/// drag/paste of image paths work across all front-ends by default. -fn attach_images_from_text(mut items: Vec, cwd: &std::path::Path) -> Vec { - use std::collections::HashSet; - use std::path::{Path, PathBuf}; +/// Configure the model session. +struct ConfigureSession { + /// Provider identifier ("openai", "openrouter", ...). + provider: ModelProviderInfo, - // Track existing attached images to avoid duplicates if the client already added some. - let mut existing: HashSet = items - .iter() - .filter_map(|it| match it { - InputItem::LocalImage { path } => Some(path.clone()), - _ => None, - }) - .collect(); + /// If not specified, server will use its default model. + model: String, - // Collect images discovered in text. - let mut discovered: Vec = Vec::new(); + model_reasoning_effort: ReasoningEffortConfig, + model_reasoning_summary: ReasoningSummaryConfig, - for it in &items { - if let InputItem::Text { text } = it { - for p in detect_image_paths_in_text(text, cwd) { - if existing.insert(p.clone()) { - discovered.push(p); - } - } - } - } + /// Model instructions that are appended to the base instructions. + user_instructions: Option, - for path in discovered { - items.push(InputItem::LocalImage { path }); - } - items -} + /// Base instructions override. + base_instructions: Option, -/// Parse a free‑form text looking for plausible image paths. -fn detect_image_paths_in_text(text: &str, cwd: &std::path::Path) -> Vec { - use std::collections::HashSet; - use std::path::{Path, PathBuf}; + /// When to escalate for approval for execution + approval_policy: AskForApproval, + /// How to sandbox commands executed in the system + sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + disable_response_storage: bool, - if text.trim().is_empty() { - return Vec::new(); - } + /// Optional external notifier command tokens. Present only when the + /// client wants the agent to spawn a program after each completed + /// turn. + notify: Option>, - // Only normalize quotes; preserve NBSP/NNBSP so we don't break real filenames. - let normalized = text - .replace('\u{2018}', "'") - .replace('\u{2019}', "'") - .replace('\u{201C}', "\"") - .replace('\u{201D}', "\""); - - // Regex to capture file:// URLs and absolute path-like substrings that end with an image extension. - let re = Regex::new(r#"(?xi)(file://\S+)|(/[^'"\s]+\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg))|([A-Za-z0-9_\-\s\.]+\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg))"#) - .ok(); - - let mut candidates: Vec = Vec::new(); - if let Some(re) = &re { - for cap in re.captures_iter(&normalized) { - if let Some(m) = cap.get(1).or_else(|| cap.get(2)).or_else(|| cap.get(3)) { - candidates.push(m.as_str().to_string()); - } - } - } + /// Working directory that should be treated as the *root* of the + /// session. All relative paths supplied by the model as well as the + /// execution sandbox are resolved against this directory **instead** + /// of the process-wide current working directory. CLI front-ends are + /// expected to expand this to an absolute path before sending the + /// `ConfigureSession` operation so that the business-logic layer can + /// operate deterministically. + cwd: PathBuf, - // Also try shlex splitting to handle quoted paths. - let mut lexer = shlex::Shlex::new(&normalized); - for tok in &mut lexer { - candidates.push(tok); - } - if candidates.is_empty() { - candidates.extend(normalized.split_whitespace().map(|s| s.to_string())); - } + resume_path: Option, +} - fn strip_surrounding_quotes(mut s: String) -> String { - if (s.starts_with('\'') && s.ends_with('\'')) || (s.starts_with('"') && s.ends_with('"')) { - s = s[1..s.len() - 1].to_string(); +impl Session { + async fn new( + configure_session: ConfigureSession, + config: Arc, + auth: Option, + tx_event: Sender, + ) -> anyhow::Result<(Arc, TurnContext)> { + let ConfigureSession { + provider, + model, + model_reasoning_effort, + model_reasoning_summary, + user_instructions, + base_instructions, + approval_policy, + sandbox_policy, + disable_response_storage, + notify, + cwd, + resume_path, + } = configure_session; + debug!("Configuring session: model={model}; provider={provider:?}"); + if !cwd.is_absolute() { + return Err(anyhow::anyhow!("cwd is not absolute: {cwd:?}")); } - s - } - fn from_file_url(url: &str) -> Option { - let rest = url.strip_prefix("file://")?; - let path = if let Some(stripped) = rest.strip_prefix("localhost") { - stripped - } else { - rest + // Error messages to dispatch after SessionConfigured is sent. + let mut post_session_configured_error_events = Vec::::new(); + + // Kick off independent async setup tasks in parallel to reduce startup latency. + // + // - initialize RolloutRecorder with new or resumed session info + // - spin up MCP connection manager + // - perform default shell discovery + // - load history metadata + let rollout_fut = async { + match resume_path.as_ref() { + Some(path) => RolloutRecorder::resume(path, cwd.clone()) + .await + .map(|(rec, saved)| (saved.session_id, Some(saved), rec)), + None => { + let session_id = Uuid::new_v4(); + RolloutRecorder::new(&config, session_id, user_instructions.clone()) + .await + .map(|rec| (session_id, None, rec)) + } + } }; - let p = path.replace("%20", " "); - if p.starts_with('/') { - Some(PathBuf::from(p)) - } else { - None - } - } - fn looks_like_image_path(p: &Path) -> bool { - match p.extension().and_then(|e| e.to_str()).map(|e| e.to_ascii_lowercase()) { - Some(ext) - if matches!( - ext.as_str(), - "png" | "jpg" | "jpeg" | "gif" | "webp" | "bmp" | "tif" | "tiff" | "svg" - ) => true, - _ => false, - } - } + let mcp_fut = McpConnectionManager::new(config.mcp_servers.clone()); + let default_shell_fut = shell::default_user_shell(); + let history_meta_fut = crate::message_history::history_metadata(&config); - fn norm_spaces(s: &str) -> String { - s.chars() - .map(|ch| match ch { - '\u{00A0}' | '\u{202F}' => ' ', - other => other, - }) - .collect::() - } + // Join all independent futures. + let (rollout_res, mcp_res, default_shell, (history_log_id, history_entry_count)) = + tokio::join!(rollout_fut, mcp_fut, default_shell_fut, history_meta_fut); - fn resolve_by_basename(basename: &str, home: &Path, cwd: &Path) -> Option { - let exact = [ - cwd.join(basename), - home.join("Desktop").join(basename), - home.join("Downloads").join(basename), - home.join("Pictures").join(basename), - home.join("Pictures").join("Screenshots").join(basename), - ]; - for p in exact.into_iter() { - if let Ok(meta) = std::fs::metadata(&p) { - if meta.is_file() { - return Some(p); + // Handle rollout result, which determines the session_id. + struct RolloutResult { + session_id: Uuid, + rollout_recorder: Option, + restored_items: Option>, + } + let rollout_result = match rollout_res { + Ok((session_id, maybe_saved, recorder)) => { + let restored_items: Option> = + maybe_saved.and_then(|saved_session| { + if saved_session.items.is_empty() { + None + } else { + Some(saved_session.items) + } + }); + RolloutResult { + session_id, + rollout_recorder: Some(recorder), + restored_items, } } - } + Err(e) => { + if let Some(path) = resume_path.as_ref() { + return Err(anyhow::anyhow!( + "failed to resume rollout from {path:?}: {e}" + )); + } - // Fallback scan with NBSP normalization for macOS screenshot names. - let norm_base = norm_spaces(basename); - let scan_dirs = [ - cwd.to_path_buf(), - home.join("Desktop"), - home.join("Downloads"), - home.join("Pictures"), - home.join("Pictures").join("Screenshots"), - ]; - for dir in scan_dirs.into_iter() { - if let Ok(rd) = std::fs::read_dir(&dir) { - for entry in rd.flatten() { - let is_file = entry.file_type().map(|ft| ft.is_file()).unwrap_or(false); - if !is_file { - continue; - } - if let Some(name) = entry.file_name().to_str() { - if norm_spaces(name) == norm_base { - return Some(entry.path()); - } - } + let message = format!("failed to initialize rollout recorder: {e}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { + message: message.clone(), + }), + }); + warn!("{message}"); + + RolloutResult { + session_id: Uuid::new_v4(), + rollout_recorder: None, + restored_items: None, } } - } - None - } + }; - let mut out: Vec = Vec::new(); - let mut seen: HashSet = HashSet::new(); + let RolloutResult { + session_id, + rollout_recorder, + restored_items, + } = rollout_result; - for raw in candidates { - if raw.is_empty() { - continue; + // Create the mutable state for the Session. + let mut state = State { + history: ConversationHistory::new(), + ..Default::default() + }; + if let Some(restored_items) = restored_items { + state.history.record_items(&restored_items); } - let candidate = if raw.starts_with("file://") { - from_file_url(&raw) - } else { - let mut tok = strip_surrounding_quotes(raw); - // Unescape typical shell-escaped spaces. - tok = tok.replace("\\ ", " "); - - // Expand ~/ - let expanded = if let Some(rest) = tok.strip_prefix("~/") { - if let Some(home) = std::env::var_os("HOME").map(PathBuf::from) { - home.join(rest) - } else { - PathBuf::from(tok) - } - } else { - PathBuf::from(tok) - }; - let abs = if expanded.is_absolute() { - expanded - } else { - cwd.join(expanded) - }; - Some(abs) + // Handle MCP manager result and record any startup failures. + let (mcp_connection_manager, failed_clients) = match mcp_res { + Ok((mgr, failures)) => (mgr, failures), + Err(e) => { + let message = format!("Failed to create MCP connection manager: {e:#}"); + error!("{message}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { message }), + }); + (McpConnectionManager::default(), Default::default()) + } }; - let Some(mut path) = candidate else { continue }; - if !looks_like_image_path(&path) { - continue; - } - let exists_and_file = std::fs::metadata(&path).map(|m| m.is_file()).unwrap_or(false); - if !exists_and_file { - if let Some(name) = path.file_name().and_then(|s| s.to_str()) { - if let Some(home) = std::env::var_os("HOME").map(PathBuf::from) { - if let Some(found) = resolve_by_basename(name, &home, cwd) { - path = found; - } else { - continue; - } - } else { - continue; - } - } else { - continue; + // Surface individual client start-up failures to the user. + if !failed_clients.is_empty() { + for (server_name, err) in failed_clients { + let message = format!("MCP client for `{server_name}` failed to start: {err:#}"); + error!("{message}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { message }), + }); } } - if seen.insert(path.clone()) { - out.push(path); + + // Now that `session_id` is final (may have been updated by resume), + // construct the model client. + let client = ModelClient::new( + config.clone(), + auth.clone(), + provider.clone(), + model_reasoning_effort, + model_reasoning_summary, + session_id, + ); + let turn_context = TurnContext { + client, + tools_config: ToolsConfig::new( + &config.model_family, + approval_policy, + sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ), + user_instructions, + base_instructions, + approval_policy, + sandbox_policy, + shell_environment_policy: config.shell_environment_policy.clone(), + cwd, + disable_response_storage, + }; + let sess = Arc::new(Session { + session_id, + tx_event: tx_event.clone(), + mcp_connection_manager, + notify, + state: Mutex::new(state), + rollout: Mutex::new(rollout_recorder), + codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), + user_shell: default_shell, + show_raw_agent_reasoning: config.show_raw_agent_reasoning, + }); + + // record the initial user instructions and environment context, + // regardless of whether we restored items. + let mut conversation_items = Vec::::with_capacity(2); + if let Some(user_instructions) = turn_context.user_instructions.as_deref() { + conversation_items.push(Prompt::format_user_instructions_message(user_instructions)); + } + conversation_items.push(ResponseItem::from(EnvironmentContext::new( + Some(turn_context.cwd.clone()), + Some(turn_context.approval_policy), + Some(turn_context.sandbox_policy.clone()), + Some(sess.user_shell.clone()), + ))); + sess.record_conversation_items(&conversation_items).await; + + // Dispatch the SessionConfiguredEvent first and then report any errors. + let events = std::iter::once(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::SessionConfigured(SessionConfiguredEvent { + session_id, + model, + history_log_id, + history_entry_count, + }), + }) + .chain(post_session_configured_error_events.into_iter()); + for event in events { + if let Err(e) = tx_event.send(event).await { + error!("failed to send event: {e:?}"); + } } - } - out -} + Ok((sess, turn_context)) + } -impl Session { pub fn set_task(&self, task: AgentTask) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if let Some(current_task) = state.current_task.take() { - current_task.abort(); + current_task.abort(TurnAbortReason::Replaced); } state.current_task = Some(task); } pub fn remove_task(&self, sub_id: &str) { - let mut state = self.state.lock().unwrap(); - if let Some(task) = &state.current_task { - if task.sub_id == sub_id { - state.current_task.take(); - } + let mut state = self.state.lock_unchecked(); + if let Some(task) = &state.current_task + && task.sub_id == sub_id + { + state.current_task.take(); } } @@ -525,7 +581,7 @@ impl Session { }; let _ = self.tx_event.send(event).await; { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.pending_approvals.insert(sub_id, tx_approve); } rx_approve @@ -551,21 +607,21 @@ impl Session { }; let _ = self.tx_event.send(event).await; { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.pending_approvals.insert(sub_id, tx_approve); } rx_approve } pub fn notify_approval(&self, sub_id: &str, decision: ReviewDecision) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if let Some(tx_approve) = state.pending_approvals.remove(sub_id) { tx_approve.send(decision).ok(); } } pub fn add_approved_command(&self, cmd: Vec) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.approved_commands.insert(cmd); } @@ -575,14 +631,14 @@ impl Session { debug!("Recording items for conversation: {items:?}"); self.record_state_snapshot(items).await; - self.state.lock().unwrap().history.record_items(items); + self.state.lock_unchecked().history.record_items(items); } async fn record_state_snapshot(&self, items: &[ResponseItem]) { let snapshot = { crate::rollout::SessionStateSnapshot {} }; let recorder = { - let guard = self.rollout.lock().unwrap(); + let guard = self.rollout.lock_unchecked(); guard.as_ref().cloned() }; @@ -625,7 +681,10 @@ impl Session { call_id, command: command_for_display.clone(), cwd, - parsed_cmd: parse_command(&command_for_display), + parsed_cmd: parse_command(&command_for_display) + .into_iter() + .map(Into::into) + .collect(), }), }; let event = Event { @@ -713,7 +772,6 @@ impl Session { let result = process_exec_tool_call( exec_args.params, exec_args.sandbox_type, - exec_args.ctrl_c, exec_args.sandbox_policy, exec_args.codex_linux_sandbox_exe, exec_args.stdout_stream, @@ -758,15 +816,25 @@ impl Session { let _ = self.tx_event.send(event).await; } + async fn notify_stream_error(&self, sub_id: &str, message: impl Into) { + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::StreamError(StreamErrorEvent { + message: message.into(), + }), + }; + let _ = self.tx_event.send(event).await; + } + /// Build the full turn input by concatenating the current conversation /// history with additional items for this turn. pub fn turn_input_with_history(&self, extra: Vec) -> Vec { - [self.state.lock().unwrap().history.contents(), extra].concat() + [self.state.lock_unchecked().history.contents(), extra].concat() } /// Returns the input if there was no task running to inject into pub fn inject_input(&self, input: Vec) -> Result<(), Vec> { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if state.current_task.is_some() { state.pending_input.push(input.into()); Ok(()) @@ -776,7 +844,7 @@ impl Session { } pub fn get_pending_input(&self) -> Vec { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if state.pending_input.is_empty() { Vec::with_capacity(0) } else { @@ -798,13 +866,13 @@ impl Session { .await } - pub fn abort(&self) { - info!("Aborting existing session"); - let mut state = self.state.lock().unwrap(); + fn interrupt_task(&self) { + info!("interrupt received: abort current task, if any"); + let mut state = self.state.lock_unchecked(); state.pending_approvals.clear(); state.pending_input.clear(); if let Some(task) = state.current_task.take() { - task.abort(); + task.abort(TurnAbortReason::Interrupted); } } @@ -840,17 +908,7 @@ impl Session { impl Drop for Session { fn drop(&mut self) { - self.abort(); - } -} - -impl State { - pub fn partial_clone(&self) -> Self { - Self { - approved_commands: self.approved_commands.clone(), - history: self.history.clone(), - ..Default::default() - } + self.interrupt_task(); } } @@ -877,28 +935,42 @@ pub(crate) struct AgentTask { } impl AgentTask { - fn spawn(sess: Arc, sub_id: String, input: Vec) -> Self { - let handle = - tokio::spawn(run_task(Arc::clone(&sess), sub_id.clone(), input)).abort_handle(); + fn spawn( + sess: Arc, + turn_context: Arc, + sub_id: String, + input: Vec, + ) -> Self { + let handle = { + let sess = sess.clone(); + let sub_id = sub_id.clone(); + let tc = Arc::clone(&turn_context); + tokio::spawn(async move { run_task(sess, tc.as_ref(), sub_id, input).await }) + .abort_handle() + }; Self { sess, sub_id, handle, } } + fn compact( sess: Arc, + turn_context: Arc, sub_id: String, input: Vec, compact_instructions: String, ) -> Self { - let handle = tokio::spawn(run_compact_task( - Arc::clone(&sess), - sub_id.clone(), - input, - compact_instructions, - )) - .abort_handle(); + let handle = { + let sess = sess.clone(); + let sub_id = sub_id.clone(); + let tc = Arc::clone(&turn_context); + tokio::spawn(async move { + run_compact_task(sess, tc.as_ref(), sub_id, input, compact_instructions).await + }) + .abort_handle() + }; Self { sess, sub_id, @@ -906,14 +978,13 @@ impl AgentTask { } } - fn abort(self) { + fn abort(self, reason: TurnAbortReason) { + // TOCTOU? if !self.handle.is_finished() { self.handle.abort(); let event = Event { id: self.sub_id, - msg: EventMsg::Error(ErrorEvent { - message: " Turn interrupted".to_string(), - }), + msg: EventMsg::TurnAborted(TurnAbortedEvent { reason }), }; let tx_event = self.sess.tx_event.clone(); tokio::spawn(async move { @@ -924,279 +995,180 @@ impl AgentTask { } async fn submission_loop( - mut session_id: Uuid, + sess: Arc, + turn_context: TurnContext, config: Arc, - auth: Option, rx_sub: Receiver, - tx_event: Sender, - ctrl_c: Arc, ) { - let mut sess: Option> = None; - // shorthand - send an event when there is no active session - let send_no_session_event = |sub_id: String| async { - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: "No session initialized, expected 'ConfigureSession' as first Op" - .to_string(), - }), - }; - tx_event.send(event).await.ok(); - }; - - loop { - let interrupted = ctrl_c.notified(); - let sub = tokio::select! { - res = rx_sub.recv() => match res { - Ok(sub) => sub, - Err(_) => break, - }, - _ = interrupted => { - if let Some(sess) = sess.as_ref(){ - sess.abort(); - } - continue; - }, - }; - + // Wrap once to avoid cloning TurnContext for each task. + let mut turn_context = Arc::new(turn_context); + // To break out of this loop, send Op::Shutdown. + while let Ok(sub) = rx_sub.recv().await { debug!(?sub, "Submission"); match sub.op { Op::Interrupt => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - sess.abort(); + sess.interrupt_task(); } - Op::ConfigureSession { - provider, - model, - model_reasoning_effort, - model_reasoning_summary, - user_instructions, - base_instructions, + Op::OverrideTurnContext { + cwd, approval_policy, sandbox_policy, - disable_response_storage, - notify, - cwd, - resume_path, + model, + effort, + summary, } => { - debug!( - "Configuring session: model={model}; provider={provider:?}; resume={resume_path:?}" - ); - if !cwd.is_absolute() { - let message = format!("cwd is not absolute: {cwd:?}"); - error!(message); - let event = Event { - id: sub.id, - msg: EventMsg::Error(ErrorEvent { message }), - }; - if let Err(e) = tx_event.send(event).await { - error!("failed to send error message: {e:?}"); - } - return; - } - // Optionally resume an existing rollout. - let mut restored_items: Option> = None; - let rollout_recorder: Option = - if let Some(path) = resume_path.as_ref() { - match RolloutRecorder::resume(path, cwd.clone()).await { - Ok((rec, saved)) => { - session_id = saved.session_id; - if !saved.items.is_empty() { - restored_items = Some(saved.items); - } - Some(rec) - } - Err(e) => { - warn!("failed to resume rollout from {path:?}: {e}"); - None - } - } - } else { - None - }; - - let rollout_recorder = match rollout_recorder { - Some(rec) => Some(rec), - None => { - match RolloutRecorder::new(&config, session_id, user_instructions.clone()) - .await - { - Ok(r) => Some(r), - Err(e) => { - warn!("failed to initialise rollout recorder: {e}"); - None - } - } - } + // Recalculate the persistent turn context with provided overrides. + let prev = Arc::clone(&turn_context); + let provider = prev.client.get_provider(); + + // Effective model + family + let (effective_model, effective_family) = if let Some(m) = model { + let fam = + find_family_for_model(&m).unwrap_or_else(|| config.model_family.clone()); + (m, fam) + } else { + (prev.client.get_model(), prev.client.get_model_family()) }; + // Effective reasoning settings + let effective_effort = effort.unwrap_or(prev.client.get_reasoning_effort()); + let effective_summary = summary.unwrap_or(prev.client.get_reasoning_summary()); + + let auth = prev.client.get_auth(); + // Build updated config for the client + let mut updated_config = (*config).clone(); + updated_config.model = effective_model.clone(); + updated_config.model_family = effective_family.clone(); + let client = ModelClient::new( - config.clone(), - auth.clone(), - provider.clone(), - model_reasoning_effort, - model_reasoning_summary, - session_id, + Arc::new(updated_config), + auth, + provider, + effective_effort, + effective_summary, + sess.session_id, ); - // abort any current running session and clone its state - let state = match sess.take() { - Some(sess) => { - sess.abort(); - sess.state.lock().unwrap().partial_clone() - } - None => State { - history: ConversationHistory::new(), - ..Default::default() - }, - }; - - let writable_roots = Mutex::new(get_writable_roots(&cwd)); - - // Error messages to dispatch after SessionConfigured is sent. - let mut mcp_connection_errors = Vec::::new(); - let (mcp_connection_manager, failed_clients) = - match McpConnectionManager::new(config.mcp_servers.clone()).await { - Ok((mgr, failures)) => (mgr, failures), - Err(e) => { - let message = format!("Failed to create MCP connection manager: {e:#}"); - error!("{message}"); - mcp_connection_errors.push(Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { message }), - }); - (McpConnectionManager::default(), Default::default()) - } - }; + let new_approval_policy = approval_policy.unwrap_or(prev.approval_policy); + let new_sandbox_policy = sandbox_policy + .clone() + .unwrap_or(prev.sandbox_policy.clone()); + let new_cwd = cwd.clone().unwrap_or_else(|| prev.cwd.clone()); + + let tools_config = ToolsConfig::new( + &effective_family, + new_approval_policy, + new_sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ); - // Surface individual client start-up failures to the user. - if !failed_clients.is_empty() { - for (server_name, err) in failed_clients { - let message = - format!("MCP client for `{server_name}` failed to start: {err:#}"); - error!("{message}"); - mcp_connection_errors.push(Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { message }), - }); - } - } - let default_shell = shell::default_user_shell().await; - sess = Some(Arc::new(Session { + let new_turn_context = TurnContext { client, - tools_config: ToolsConfig::new( - &config.model_family, - approval_policy, - sandbox_policy.clone(), - config.include_plan_tool, - ), - tx_event: tx_event.clone(), - ctrl_c: Arc::clone(&ctrl_c), - user_instructions, - base_instructions, - approval_policy, - sandbox_policy, - shell_environment_policy: config.shell_environment_policy.clone(), - cwd, - writable_roots, - mcp_connection_manager, - notify, - state: Mutex::new(state), - rollout: Mutex::new(rollout_recorder), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - disable_response_storage, - user_shell: default_shell, - show_raw_agent_reasoning: config.show_raw_agent_reasoning, - })); - - // Patch restored state into the newly created session. - if let Some(sess_arc) = &sess { - if restored_items.is_some() { - let mut st = sess_arc.state.lock().unwrap(); - st.history.record_items(restored_items.unwrap().iter()); - } - } - - // Gather history metadata for SessionConfiguredEvent. - let (history_log_id, history_entry_count) = - crate::message_history::history_metadata(&config).await; + tools_config, + user_instructions: prev.user_instructions.clone(), + base_instructions: prev.base_instructions.clone(), + approval_policy: new_approval_policy, + sandbox_policy: new_sandbox_policy.clone(), + shell_environment_policy: prev.shell_environment_policy.clone(), + cwd: new_cwd.clone(), + disable_response_storage: prev.disable_response_storage, + }; - // ack - let events = std::iter::once(Event { - id: sub.id.clone(), - msg: EventMsg::SessionConfigured(SessionConfiguredEvent { - session_id, - model, - history_log_id, - history_entry_count, - }), - }) - .chain(mcp_connection_errors.into_iter()); - for event in events { - if let Err(e) = tx_event.send(event).await { - error!("failed to send event: {e:?}"); - } + // Install the new persistent context for subsequent tasks/turns. + turn_context = Arc::new(new_turn_context); + if cwd.is_some() || approval_policy.is_some() || sandbox_policy.is_some() { + sess.record_conversation_items(&[ResponseItem::from(EnvironmentContext::new( + cwd, + approval_policy, + sandbox_policy, + // Shell is not configurable from turn to turn + None, + ))]) + .await; } } Op::UserInput { items } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - - // Auto‑attach local images referenced in text items so mid‑chat drags work by default. - let items = attach_images_from_text(items, &sess.cwd); - // attempt to inject input into current task if let Err(items) = sess.inject_input(items) { // no current task, spawn a new one - let task = AgentTask::spawn(Arc::clone(sess), sub.id, items); + let task = + AgentTask::spawn(sess.clone(), Arc::clone(&turn_context), sub.id, items); sess.set_task(task); } } - Op::ExecApproval { id, decision } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - match decision { - ReviewDecision::Abort => { - sess.abort(); - } - other => sess.notify_approval(&id, other), + Op::UserTurn { + items, + cwd, + approval_policy, + sandbox_policy, + model, + effort, + summary, + } => { + // attempt to inject input into current task + if let Err(items) = sess.inject_input(items) { + // Derive a fresh TurnContext for this turn using the provided overrides. + let provider = turn_context.client.get_provider(); + + // Derive a model family for the requested model; fall back to the session's. + let model_family = find_family_for_model(&model) + .unwrap_or_else(|| config.model_family.clone()); + + // Create a per‑turn Config clone with the requested model/family. + let mut per_turn_config = (*config).clone(); + per_turn_config.model = model.clone(); + per_turn_config.model_family = model_family.clone(); + + // Build a new client with per‑turn reasoning settings. + // Reuse the same provider and session id; auth defaults to env/API key. + let client = ModelClient::new( + Arc::new(per_turn_config), + None, + provider, + effort, + summary, + sess.session_id, + ); + + let fresh_turn_context = TurnContext { + client, + tools_config: ToolsConfig::new( + &model_family, + approval_policy, + sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ), + user_instructions: turn_context.user_instructions.clone(), + base_instructions: turn_context.base_instructions.clone(), + approval_policy, + sandbox_policy, + shell_environment_policy: turn_context.shell_environment_policy.clone(), + cwd, + disable_response_storage: turn_context.disable_response_storage, + }; + // TODO: record the new environment context in the conversation history + // no current task, spawn a new one with the per‑turn context + let task = + AgentTask::spawn(sess.clone(), Arc::new(fresh_turn_context), sub.id, items); + sess.set_task(task); } } - Op::PatchApproval { id, decision } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - match decision { - ReviewDecision::Abort => { - sess.abort(); - } - other => sess.notify_approval(&id, other), + Op::ExecApproval { id, decision } => match decision { + ReviewDecision::Abort => { + sess.interrupt_task(); } - } + other => sess.notify_approval(&id, other), + }, + Op::PatchApproval { id, decision } => match decision { + ReviewDecision::Abort => { + sess.interrupt_task(); + } + other => sess.notify_approval(&id, other), + }, Op::AddToHistory { text } => { - // TODO: What should we do if we got AddToHistory before ConfigureSession? - // currently, if ConfigureSession has resume path, this history will be ignored - let id = session_id; + let id = sess.session_id; let config = config.clone(); tokio::spawn(async move { if let Err(e) = crate::message_history::append_entry(&text, &id, &config).await @@ -1208,7 +1180,7 @@ async fn submission_loop( Op::GetHistoryEntryRequest { offset, log_id } => { let config = config.clone(); - let tx_event = tx_event.clone(); + let tx_event = sess.tx_event.clone(); let sub_id = sub.id.clone(); tokio::spawn(async move { @@ -1225,7 +1197,13 @@ async fn submission_loop( crate::protocol::GetHistoryEntryResponseEvent { offset, log_id, - entry: entry_opt, + entry: entry_opt.map(|e| { + codex_protocol::message_history::HistoryEntry { + session_id: e.session_id, + ts: e.ts, + text: e.text, + } + }), }, ), }; @@ -1235,15 +1213,23 @@ async fn submission_loop( } }); } - Op::Compact => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; + Op::ListMcpTools => { + let tx_event = sess.tx_event.clone(); + let sub_id = sub.id.clone(); + // This is a cheap lookup from the connection manager's cache. + let tools = sess.mcp_connection_manager.list_all_tools(); + let event = Event { + id: sub_id, + msg: EventMsg::McpListToolsResponse( + crate::protocol::McpListToolsResponseEvent { tools }, + ), + }; + if let Err(e) = tx_event.send(event).await { + warn!("failed to send McpListToolsResponse event: {e}"); + } + } + Op::Compact => { // Create a summarization request as user input const SUMMARIZATION_PROMPT: &str = include_str!("prompt_for_compact_command.md"); @@ -1253,6 +1239,7 @@ async fn submission_loop( }]) { let task = AgentTask::compact( sess.clone(), + Arc::clone(&turn_context), sub.id, items, SUMMARIZATION_PROMPT.to_string(), @@ -1265,32 +1252,34 @@ async fn submission_loop( // Gracefully flush and shutdown rollout recorder on session end so tests // that inspect the rollout file do not race with the background writer. - if let Some(sess_arc) = sess { - let recorder_opt = sess_arc.rollout.lock().unwrap().take(); - if let Some(rec) = recorder_opt { - if let Err(e) = rec.shutdown().await { - warn!("failed to shutdown rollout recorder: {e}"); - let event = Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: "Failed to shutdown rollout recorder".to_string(), - }), - }; - if let Err(e) = tx_event.send(event).await { - warn!("failed to send error message: {e:?}"); - } - } + let recorder_opt = sess.rollout.lock_unchecked().take(); + if let Some(rec) = recorder_opt + && let Err(e) = rec.shutdown().await + { + warn!("failed to shutdown rollout recorder: {e}"); + let event = Event { + id: sub.id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: "Failed to shutdown rollout recorder".to_string(), + }), + }; + if let Err(e) = sess.tx_event.send(event).await { + warn!("failed to send error message: {e:?}"); } } + let event = Event { id: sub.id.clone(), msg: EventMsg::ShutdownComplete, }; - if let Err(e) = tx_event.send(event).await { + if let Err(e) = sess.tx_event.send(event).await { warn!("failed to send Shutdown event: {e}"); } break; } + _ => { + // Ignore unknown ops; enum is non_exhaustive to allow extensions. + } } } debug!("Agent loop exited"); @@ -1309,7 +1298,12 @@ async fn submission_loop( /// back to the model in the next turn. /// - If the model sends only an assistant message, we record it in the /// conversation history and consider the task complete. -async fn run_task(sess: Arc, sub_id: String, input: Vec) { +async fn run_task( + sess: Arc, + turn_context: &TurnContext, + sub_id: String, + input: Vec, +) { if input.is_empty() { return; } @@ -1361,7 +1355,15 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { }) }) .collect(); - match run_turn(&sess, &mut turn_diff_tracker, sub_id.clone(), turn_input).await { + match run_turn( + &sess, + turn_context, + &mut turn_diff_tracker, + sub_id.clone(), + turn_input, + ) + .await + { Ok(turn_output) => { let mut items_to_record_in_conversation_history = Vec::::new(); let mut responses = Vec::::new(); @@ -1490,31 +1492,26 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, input: Vec, ) -> CodexResult> { let tools = get_openai_tools( - &sess.tools_config, + &turn_context.tools_config, Some(sess.mcp_connection_manager.list_all_tools()), ); let prompt = Prompt { input, - user_instructions: sess.user_instructions.clone(), - store: !sess.disable_response_storage, + store: !turn_context.disable_response_storage, tools, - base_instructions_override: sess.base_instructions.clone(), - environment_context: Some(EnvironmentContext { - cwd: sess.cwd.clone(), - approval_policy: sess.approval_policy, - sandbox_policy: sess.sandbox_policy.clone(), - }), + base_instructions_override: turn_context.base_instructions.clone(), }; let mut retries = 0; loop { - match try_run_turn(sess, turn_diff_tracker, &sub_id, &prompt).await { + match try_run_turn(sess, turn_context, turn_diff_tracker, &sub_id, &prompt).await { Ok(output) => return Ok(output), Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted), Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)), @@ -1523,10 +1520,13 @@ async fn run_turn( } Err(e) => { // Use the configured provider-specific stream retry budget. - let max_retries = sess.client.get_provider().stream_max_retries(); + let max_retries = turn_context.client.get_provider().stream_max_retries(); if retries < max_retries { retries += 1; - let delay = backoff(retries); + let delay = match e { + CodexErr::Stream(_, Some(delay)) => delay, + _ => backoff(retries), + }; warn!( "stream disconnected - retrying turn ({retries}/{max_retries} in {delay:?})...", ); @@ -1534,7 +1534,7 @@ async fn run_turn( // Surface retry information to any UI/front‑end so the // user understands what is happening instead of staring // at a seemingly frozen screen. - sess.notify_background_event( + sess.notify_stream_error( &sub_id, format!( "stream error: {e}; retrying {retries}/{max_retries} in {delay:?}…" @@ -1563,6 +1563,7 @@ struct ProcessedResponseItem { async fn try_run_turn( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, prompt: &Prompt, @@ -1623,7 +1624,7 @@ async fn try_run_turn( }) }; - let mut stream = sess.client.clone().stream(&prompt).await?; + let mut stream = turn_context.client.clone().stream(&prompt).await?; let mut output = Vec::new(); loop { @@ -1636,6 +1637,7 @@ async fn try_run_turn( // Treat as a disconnected stream so the caller can retry. return Err(CodexErr::Stream( "stream closed before response.completed".into(), + None, )); }; @@ -1651,9 +1653,14 @@ async fn try_run_turn( match event { ResponseEvent::Created => {} ResponseEvent::OutputItemDone(item) => { - let response = - handle_response_item(sess, turn_diff_tracker, sub_id, item.clone()).await?; - + let response = handle_response_item( + sess, + turn_context, + turn_diff_tracker, + sub_id, + item.clone(), + ) + .await?; output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { @@ -1684,7 +1691,7 @@ async fn try_run_turn( } ResponseEvent::OutputTextDelta(delta) => { { - let mut st = sess.state.lock().unwrap(); + let mut st = sess.state.lock_unchecked(); st.history.append_assistant_text(&delta); } @@ -1725,6 +1732,7 @@ async fn try_run_turn( async fn run_compact_task( sess: Arc, + turn_context: &TurnContext, sub_id: String, input: Vec, compact_instructions: String, @@ -1743,18 +1751,16 @@ async fn run_compact_task( let prompt = Prompt { input: turn_input, - user_instructions: None, - store: !sess.disable_response_storage, - environment_context: None, + store: !turn_context.disable_response_storage, tools: Vec::new(), base_instructions_override: Some(compact_instructions.clone()), }; - let max_retries = sess.client.get_provider().stream_max_retries(); + let max_retries = turn_context.client.get_provider().stream_max_retries(); let mut retries = 0; loop { - let attempt_result = drain_to_completed(&sess, &sub_id, &prompt).await; + let attempt_result = drain_to_completed(&sess, turn_context, &sub_id, &prompt).await; match attempt_result { Ok(()) => break, @@ -1763,7 +1769,7 @@ async fn run_compact_task( if retries < max_retries { retries += 1; let delay = backoff(retries); - sess.notify_background_event( + sess.notify_stream_error( &sub_id, format!( "stream error: {e}; retrying {retries}/{max_retries} in {delay:?}…" @@ -1802,12 +1808,13 @@ async fn run_compact_task( }; sess.send_event(event).await; - let mut state = sess.state.lock().unwrap(); + let mut state = sess.state.lock_unchecked(); state.history.keep_last_messages(1); } async fn handle_response_item( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, item: ResponseItem, @@ -1842,11 +1849,13 @@ async fn handle_response_item( }; sess.tx_event.send(event).await.ok(); } - if sess.show_raw_agent_reasoning && content.is_some() { - let content = content.unwrap(); + if sess.show_raw_agent_reasoning + && let Some(content) = content + { for item in content { let text = match item { ReasoningItemContent::ReasoningText { text } => text, + ReasoningItemContent::Text { text } => text, }; let event = Event { id: sub_id.to_string(), @@ -1869,6 +1878,7 @@ async fn handle_response_item( Some( handle_function_call( sess, + turn_context, turn_diff_tracker, sub_id.to_string(), name, @@ -1908,11 +1918,12 @@ async fn handle_response_item( } }; - let exec_params = to_exec_params(params, sess); + let exec_params = to_exec_params(params, turn_context); Some( handle_container_exec_with_params( exec_params, sess, + turn_context, turn_diff_tracker, sub_id.to_string(), effective_call_id, @@ -1931,6 +1942,7 @@ async fn handle_response_item( async fn handle_function_call( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, name: String, @@ -1939,14 +1951,52 @@ async fn handle_function_call( ) -> ResponseInputItem { match name.as_str() { "container.exec" | "shell" => { - let params = match parse_container_exec_arguments(arguments, sess, &call_id) { + let params = match parse_container_exec_arguments(arguments, turn_context, &call_id) { Ok(params) => params, Err(output) => { return *output; } }; - handle_container_exec_with_params(params, sess, turn_diff_tracker, sub_id, call_id) - .await + handle_container_exec_with_params( + params, + sess, + turn_context, + turn_diff_tracker, + sub_id, + call_id, + ) + .await + } + "apply_patch" => { + let args = match serde_json::from_str::(&arguments) { + Ok(a) => a, + Err(e) => { + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!("failed to parse function arguments: {e}"), + success: None, + }, + }; + } + }; + let exec_params = ExecParams { + command: vec!["apply_patch".to_string(), args.input.clone()], + cwd: turn_context.cwd.clone(), + timeout_ms: None, + env: HashMap::new(), + with_escalated_permissions: None, + justification: None, + }; + handle_container_exec_with_params( + exec_params, + sess, + turn_context, + turn_diff_tracker, + sub_id, + call_id, + ) + .await } "update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await, _ => { @@ -1974,12 +2024,12 @@ async fn handle_function_call( } } -fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams { +fn to_exec_params(params: ShellToolCallParams, turn_context: &TurnContext) -> ExecParams { ExecParams { command: params.command, - cwd: sess.resolve_path(params.workdir.clone()), + cwd: turn_context.resolve_path(params.workdir.clone()), timeout_ms: params.timeout_ms, - env: create_env(&sess.shell_environment_policy), + env: create_env(&turn_context.shell_environment_policy), with_escalated_permissions: params.with_escalated_permissions, justification: params.justification, } @@ -1987,12 +2037,12 @@ fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams { fn parse_container_exec_arguments( arguments: String, - sess: &Session, + turn_context: &TurnContext, call_id: &str, ) -> Result> { // parse command match serde_json::from_str::(&arguments) { - Ok(shell_tool_call_params) => Ok(to_exec_params(shell_tool_call_params, sess)), + Ok(shell_tool_call_params) => Ok(to_exec_params(shell_tool_call_params, turn_context)), Err(e) => { // allow model to re-sample let output = ResponseInputItem::FunctionCallOutput { @@ -2010,20 +2060,25 @@ fn parse_container_exec_arguments( pub struct ExecInvokeArgs<'a> { pub params: ExecParams, pub sandbox_type: SandboxType, - pub ctrl_c: Arc, pub sandbox_policy: &'a SandboxPolicy, pub codex_linux_sandbox_exe: &'a Option, pub stdout_stream: Option, } -fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams { - if sess.shell_environment_policy.use_profile { - let command = sess +fn maybe_translate_shell_command( + params: ExecParams, + sess: &Session, + turn_context: &TurnContext, +) -> ExecParams { + let should_translate = matches!(sess.user_shell, crate::shell::Shell::PowerShell(_)) + || turn_context.shell_environment_policy.use_profile; + + if should_translate + && let Some(command) = sess .user_shell - .format_default_shell_invocation(params.command.clone()); - if let Some(command) = command { - return ExecParams { command, ..params }; - } + .format_default_shell_invocation(params.command.clone()) + { + return ExecParams { command, ..params }; } params } @@ -2031,6 +2086,7 @@ fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams async fn handle_container_exec_with_params( params: ExecParams, sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, call_id: String, @@ -2038,7 +2094,7 @@ async fn handle_container_exec_with_params( // check if this was a patch, and apply it if so let apply_patch_exec = match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { MaybeApplyPatchVerified::Body(changes) => { - match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await { + match apply_patch::apply_patch(sess, turn_context, &sub_id, &call_id, changes).await { InternalApplyPatchInvocation::Output(item) => return item, InternalApplyPatchInvocation::DelegateToExec(apply_patch_exec) => { Some(apply_patch_exec) @@ -2100,8 +2156,8 @@ async fn handle_container_exec_with_params( } } else { assess_safety_for_untrusted_command( - sess.approval_policy, - &sess.sandbox_policy, + turn_context.approval_policy, + &turn_context.sandbox_policy, params.with_escalated_permissions.unwrap_or(false), ) }; @@ -2113,11 +2169,11 @@ async fn handle_container_exec_with_params( } None => { let safety = { - let state = sess.state.lock().unwrap(); + let state = sess.state.lock_unchecked(); assess_command_safety( ¶ms.command, - sess.approval_policy, - &sess.sandbox_policy, + turn_context.approval_policy, + &turn_context.sandbox_policy, &state.approved_commands, params.with_escalated_permissions.unwrap_or(false), ) @@ -2187,7 +2243,7 @@ async fn handle_container_exec_with_params( ), }; - let params = maybe_run_with_user_profile(params, sess); + let params = maybe_translate_shell_command(params, sess, turn_context); let output_result = sess .run_exec_with_events( turn_diff_tracker, @@ -2195,8 +2251,7 @@ async fn handle_container_exec_with_params( ExecInvokeArgs { params: params.clone(), sandbox_type, - ctrl_c: sess.ctrl_c.clone(), - sandbox_policy: &sess.sandbox_policy, + sandbox_policy: &turn_context.sandbox_policy, codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe, stdout_stream: Some(StdoutStream { sub_id: sub_id.clone(), @@ -2229,6 +2284,7 @@ async fn handle_container_exec_with_params( error, sandbox_type, sess, + turn_context, ) .await } @@ -2249,6 +2305,7 @@ async fn handle_sandbox_error( error: SandboxErr, sandbox_type: SandboxType, sess: &Session, + turn_context: &TurnContext, ) -> ResponseInputItem { let call_id = exec_command_context.call_id.clone(); let sub_id = exec_command_context.sub_id.clone(); @@ -2256,7 +2313,7 @@ async fn handle_sandbox_error( // Early out if either the user never wants to be asked for approval, or // we're letting the model manage escalation requests. Otherwise, continue - match sess.approval_policy { + match turn_context.approval_policy { AskForApproval::Never | AskForApproval::OnRequest => { return ResponseInputItem::FunctionCallOutput { call_id, @@ -2327,8 +2384,7 @@ async fn handle_sandbox_error( ExecInvokeArgs { params, sandbox_type: SandboxType::None, - ctrl_c: sess.ctrl_c.clone(), - sandbox_policy: &sess.sandbox_policy, + sandbox_policy: &turn_context.sandbox_policy, codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe, stdout_stream: Some(StdoutStream { sub_id: sub_id.clone(), @@ -2442,19 +2498,25 @@ fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option CodexResult<()> { - let mut stream = sess.client.clone().stream(prompt).await?; +async fn drain_to_completed( + sess: &Session, + turn_context: &TurnContext, + sub_id: &str, + prompt: &Prompt, +) -> CodexResult<()> { + let mut stream = turn_context.client.clone().stream(prompt).await?; loop { let maybe_event = stream.next().await; let Some(event) = maybe_event else { return Err(CodexErr::Stream( "stream closed before response.completed".into(), + None, )); }; match event { Ok(ResponseEvent::OutputItemDone(item)) => { // Record only to in-memory conversation history; avoid state snapshot. - let mut state = sess.state.lock().unwrap(); + let mut state = sess.state.lock_unchecked(); state.history.record_items(std::slice::from_ref(&item)); } Ok(ResponseEvent::Completed { @@ -2466,6 +2528,7 @@ async fn drain_to_completed(sess: &Session, sub_id: &str, prompt: &Prompt) -> Co None => { return Err(CodexErr::Stream( "token_usage was None in ResponseEvent::Completed".into(), + None, )); } }; diff --git a/codex-rs/core/src/codex_conversation.rs b/codex-rs/core/src/codex_conversation.rs new file mode 100644 index 00000000000..d3b00046fc0 --- /dev/null +++ b/codex-rs/core/src/codex_conversation.rs @@ -0,0 +1,30 @@ +use crate::codex::Codex; +use crate::error::Result as CodexResult; +use crate::protocol::Event; +use crate::protocol::Op; +use crate::protocol::Submission; + +pub struct CodexConversation { + codex: Codex, +} + +/// Conduit for the bidirectional stream of messages that compose a conversation +/// in Codex. +impl CodexConversation { + pub(crate) fn new(codex: Codex) -> Self { + Self { codex } + } + + pub async fn submit(&self, op: Op) -> CodexResult { + self.codex.submit(op).await + } + + /// Use sparingly: this is intended to be removed soon. + pub async fn submit_with_id(&self, sub: Submission) -> CodexResult<()> { + self.codex.submit_with_id(sub).await + } + + pub async fn next_event(&self) -> CodexResult { + self.codex.next_event().await + } +} diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs deleted file mode 100644 index dc10ec8d847..00000000000 --- a/codex-rs/core/src/codex_wrapper.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::sync::Arc; - -use crate::Codex; -use crate::CodexSpawnOk; -use crate::config::Config; -use crate::protocol::Event; -use crate::protocol::EventMsg; -use crate::util::notify_on_sigint; -use codex_login::CodexAuth; -use tokio::sync::Notify; -use uuid::Uuid; - -/// Represents an active Codex conversation, including the first event -/// (which is [`EventMsg::SessionConfigured`]). -pub struct CodexConversation { - pub codex: Codex, - pub session_id: Uuid, - pub session_configured: Event, - pub ctrl_c: Arc, -} - -/// Spawn a new [`Codex`] and initialize the session. -/// -/// Returns the wrapped [`Codex`] **and** the `SessionInitialized` event that -/// is received as a response to the initial `ConfigureSession` submission so -/// that callers can surface the information to the UI. -pub async fn init_codex(config: Config) -> anyhow::Result { - let ctrl_c = notify_on_sigint(); - let auth = CodexAuth::from_codex_home(&config.codex_home)?; - let CodexSpawnOk { - codex, - init_id, - session_id, - } = Codex::spawn(config, auth, ctrl_c.clone()).await?; - - // The first event must be `SessionInitialized`. Validate and forward it to - // the caller so that they can display it in the conversation history. - let event = codex.next_event().await?; - if event.id != init_id - || !matches!( - &event, - Event { - id: _id, - msg: EventMsg::SessionConfigured(_), - } - ) - { - return Err(anyhow::anyhow!( - "expected SessionInitialized but got {event:?}" - )); - } - - Ok(CodexConversation { - codex, - session_id, - session_configured: event, - ctrl_c, - }) -} diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index f9c15b9eed8..4b6f8ac9ef5 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,9 +1,6 @@ use crate::config_profile::ConfigProfile; use crate::config_types::History; use crate::config_types::McpServerConfig; -use crate::config_types::ReasoningEffort; -use crate::config_types::ReasoningSummary; -use crate::config_types::SandboxMode; use crate::config_types::SandboxWorkspaceWrite; use crate::config_types::ShellEnvironmentPolicy; use crate::config_types::ShellEnvironmentPolicyToml; @@ -16,6 +13,10 @@ use crate::model_provider_info::built_in_model_providers; use crate::openai_model_info::get_model_info; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; +use codex_login::AuthMode; +use codex_protocol::config_types::ReasoningEffort; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::SandboxMode; use dirs::home_dir; use serde::Deserialize; use std::collections::HashMap; @@ -34,6 +35,8 @@ pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB const CONFIG_TOML_FILE: &str = "config.toml"; +const DEFAULT_RESPONSES_ORIGINATOR_HEADER: &str = "codex_cli_rs"; + /// Application configuration loaded from disk and merged with overrides. #[derive(Debug, Clone, PartialEq)] pub struct Config { @@ -139,8 +142,8 @@ pub struct Config { /// When this program is invoked, arg0 will be set to `codex-linux-sandbox`. pub codex_linux_sandbox_exe: Option, - /// If not "none", the value to use for `reasoning.effort` when making a - /// request using the Responses API. + /// Value to use for `reasoning.effort` when making a request using the + /// Responses API. pub model_reasoning_effort: ReasoningEffort, /// If not "none", the value to use for `reasoning.summary` when making a @@ -156,8 +159,16 @@ pub struct Config { /// Include an experimental plan tool that the model can use to update its current plan and status of each step. pub include_plan_tool: bool, + /// Include the `apply_patch` tool for models that benefit from invoking + /// file edits as a structured tool call. When unset, this falls back to the + /// model family's default preference. + pub include_apply_patch_tool: bool, + /// The value for the `originator` header included with Responses API requests. - pub internal_originator: Option, + pub responses_originator_header: String, + + /// If set to `true`, the API key will be signed with the `originator` header. + pub preferred_auth_method: AuthMode, } impl Config { @@ -401,9 +412,12 @@ pub struct ConfigToml { pub experimental_instructions_file: Option, /// The value for the `originator` header included with Responses API requests. - pub internal_originator: Option, + pub responses_originator_header_internal_override: Option, pub projects: Option>, + + /// If set to `true`, the API key will be signed with the `originator` header. + pub preferred_auth_method: Option, } #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] @@ -480,6 +494,7 @@ pub struct ConfigOverrides { pub codex_linux_sandbox_exe: Option, pub base_instructions: Option, pub include_plan_tool: Option, + pub include_apply_patch_tool: Option, pub disable_response_storage: Option, pub show_raw_agent_reasoning: Option, } @@ -505,6 +520,7 @@ impl Config { codex_linux_sandbox_exe, base_instructions, include_plan_tool, + include_apply_patch_tool, disable_response_storage, show_raw_agent_reasoning, } = overrides; @@ -581,6 +597,7 @@ impl Config { needs_special_apply_patch_instructions: false, supports_reasoning_summaries, uses_local_shell_tool: false, + uses_apply_patch_tool: false, } }); @@ -607,6 +624,13 @@ impl Config { Self::get_base_instructions(experimental_instructions_path, &resolved_cwd)?; let base_instructions = base_instructions.or(file_base_instructions); + let include_apply_patch_tool_val = + include_apply_patch_tool.unwrap_or(model_family.uses_apply_patch_tool); + + let responses_originator_header: String = cfg + .responses_originator_header_internal_override + .unwrap_or(DEFAULT_RESPONSES_ORIGINATOR_HEADER.to_owned()); + let config = Self { model, model_family, @@ -659,7 +683,9 @@ impl Config { experimental_resume, include_plan_tool: include_plan_tool.unwrap_or(false), - internal_originator: cfg.internal_originator, + include_apply_patch_tool: include_apply_patch_tool_val, + responses_originator_header, + preferred_auth_method: cfg.preferred_auth_method.unwrap_or(AuthMode::ChatGPT), }; Ok(config) } @@ -739,10 +765,10 @@ fn default_model() -> String { pub fn find_codex_home() -> std::io::Result { // Honor the `CODEX_HOME` environment variable when it is set to allow users // (and tests) to override the default location. - if let Ok(val) = std::env::var("CODEX_HOME") { - if !val.is_empty() { - return PathBuf::from(val).canonicalize(); - } + if let Ok(val) = std::env::var("CODEX_HOME") + && !val.is_empty() + { + return PathBuf::from(val).canonicalize(); } let mut p = home_dir().ok_or_else(|| { @@ -765,7 +791,6 @@ pub fn log_dir(cfg: &Config) -> std::io::Result { #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] use crate::config_types::HistoryPersistence; use super::*; @@ -1023,7 +1048,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, - internal_originator: None, + include_apply_patch_tool: false, + responses_originator_header: "codex_cli_rs".to_string(), + preferred_auth_method: AuthMode::ChatGPT, }, o3_profile_config ); @@ -1074,7 +1101,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, - internal_originator: None, + include_apply_patch_tool: false, + responses_originator_header: "codex_cli_rs".to_string(), + preferred_auth_method: AuthMode::ChatGPT, }; assert_eq!(expected_gpt3_profile_config, gpt3_profile_config); @@ -1140,7 +1169,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, - internal_originator: None, + include_apply_patch_tool: false, + responses_originator_header: "codex_cli_rs".to_string(), + preferred_auth_method: AuthMode::ChatGPT, }; assert_eq!(expected_zdr_profile_config, zdr_profile_config); diff --git a/codex-rs/core/src/config_profile.rs b/codex-rs/core/src/config_profile.rs index d945d1df7a4..8bb0738788b 100644 --- a/codex-rs/core/src/config_profile.rs +++ b/codex-rs/core/src/config_profile.rs @@ -1,9 +1,9 @@ use serde::Deserialize; use std::path::PathBuf; -use crate::config_types::ReasoningEffort; -use crate::config_types::ReasoningSummary; use crate::protocol::AskForApproval; +use codex_protocol::config_types::ReasoningEffort; +use codex_protocol::config_types::ReasoningSummary; /// Collection of common configuration options that a user can define as a unit /// in `config.toml`. diff --git a/codex-rs/core/src/config_types.rs b/codex-rs/core/src/config_types.rs index 291dcb6422a..d6661699fea 100644 --- a/codex-rs/core/src/config_types.rs +++ b/codex-rs/core/src/config_types.rs @@ -5,11 +5,9 @@ use std::collections::HashMap; use std::path::PathBuf; -use strum_macros::Display; use wildmatch::WildMatchPattern; use serde::Deserialize; -use serde::Serialize; #[derive(Deserialize, Debug, Clone, PartialEq)] pub struct McpServerConfig { @@ -78,20 +76,6 @@ pub enum HistoryPersistence { #[derive(Deserialize, Debug, Clone, PartialEq, Default)] pub struct Tui {} -#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Default, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum SandboxMode { - #[serde(rename = "read-only")] - #[default] - ReadOnly, - - #[serde(rename = "workspace-write")] - WorkspaceWrite, - - #[serde(rename = "danger-full-access")] - DangerFullAccess, -} - #[derive(Deserialize, Debug, Clone, PartialEq, Default)] pub struct SandboxWorkspaceWrite { #[serde(default)] @@ -199,31 +183,3 @@ impl From for ShellEnvironmentPolicy { } } } - -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum ReasoningEffort { - Low, - #[default] - Medium, - High, - /// Option to disable reasoning. - None, -} - -/// A summary of the reasoning performed by the model. This can be useful for -/// debugging and understanding the model's reasoning process. -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum ReasoningSummary { - #[default] - Auto, - Concise, - Detailed, - /// Option to disable reasoning summaries. - None, -} diff --git a/codex-rs/core/src/conversation_manager.rs b/codex-rs/core/src/conversation_manager.rs new file mode 100644 index 00000000000..2dc69be45ab --- /dev/null +++ b/codex-rs/core/src/conversation_manager.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use codex_login::CodexAuth; +use tokio::sync::RwLock; +use uuid::Uuid; + +use crate::codex::Codex; +use crate::codex::CodexSpawnOk; +use crate::codex::INITIAL_SUBMIT_ID; +use crate::codex_conversation::CodexConversation; +use crate::config::Config; +use crate::error::CodexErr; +use crate::error::Result as CodexResult; +use crate::protocol::Event; +use crate::protocol::EventMsg; +use crate::protocol::SessionConfiguredEvent; + +/// Represents a newly created Codex conversation, including the first event +/// (which is [`EventMsg::SessionConfigured`]). +pub struct NewConversation { + pub conversation_id: Uuid, + pub conversation: Arc, + pub session_configured: SessionConfiguredEvent, +} + +/// [`ConversationManager`] is responsible for creating conversations and +/// maintaining them in memory. +pub struct ConversationManager { + conversations: Arc>>>, +} + +impl Default for ConversationManager { + fn default() -> Self { + Self { + conversations: Arc::new(RwLock::new(HashMap::new())), + } + } +} + +impl ConversationManager { + pub async fn new_conversation(&self, config: Config) -> CodexResult { + let auth = CodexAuth::from_codex_home(&config.codex_home, config.preferred_auth_method)?; + self.new_conversation_with_auth(config, auth).await + } + + /// Used for integration tests: should not be used by ordinary business + /// logic. + pub async fn new_conversation_with_auth( + &self, + config: Config, + auth: Option, + ) -> CodexResult { + let CodexSpawnOk { + codex, + session_id: conversation_id, + } = Codex::spawn(config, auth).await?; + + // The first event must be `SessionInitialized`. Validate and forward it + // to the caller so that they can display it in the conversation + // history. + let event = codex.next_event().await?; + let session_configured = match event { + Event { + id, + msg: EventMsg::SessionConfigured(session_configured), + } if id == INITIAL_SUBMIT_ID => session_configured, + _ => { + return Err(CodexErr::SessionConfiguredNotFirstEvent); + } + }; + + let conversation = Arc::new(CodexConversation::new(codex)); + self.conversations + .write() + .await + .insert(conversation_id, conversation.clone()); + + Ok(NewConversation { + conversation_id, + conversation, + session_configured, + }) + } + + pub async fn get_conversation( + &self, + conversation_id: Uuid, + ) -> CodexResult> { + let conversations = self.conversations.read().await; + conversations + .get(&conversation_id) + .cloned() + .ok_or_else(|| CodexErr::ConversationNotFound(conversation_id)) + } +} diff --git a/codex-rs/core/src/environment_context.rs b/codex-rs/core/src/environment_context.rs new file mode 100644 index 00000000000..c89d7ca6c70 --- /dev/null +++ b/codex-rs/core/src/environment_context.rs @@ -0,0 +1,121 @@ +use serde::Deserialize; +use serde::Serialize; +use strum_macros::Display as DeriveDisplay; + +use crate::models::ContentItem; +use crate::models::ResponseItem; +use crate::protocol::AskForApproval; +use crate::protocol::SandboxPolicy; +use crate::shell::Shell; +use codex_protocol::config_types::SandboxMode; +use std::path::PathBuf; + +/// wraps environment context message in a tag for the model to parse more easily. +pub(crate) const ENVIRONMENT_CONTEXT_START: &str = ""; +pub(crate) const ENVIRONMENT_CONTEXT_END: &str = ""; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, DeriveDisplay)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum NetworkAccess { + Restricted, + Enabled, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename = "environment_context", rename_all = "snake_case")] +pub(crate) struct EnvironmentContext { + pub cwd: Option, + pub approval_policy: Option, + pub sandbox_mode: Option, + pub network_access: Option, + pub shell: Option, +} + +impl EnvironmentContext { + pub fn new( + cwd: Option, + approval_policy: Option, + sandbox_policy: Option, + shell: Option, + ) -> Self { + Self { + cwd, + approval_policy, + sandbox_mode: match sandbox_policy { + Some(SandboxPolicy::DangerFullAccess) => Some(SandboxMode::DangerFullAccess), + Some(SandboxPolicy::ReadOnly) => Some(SandboxMode::ReadOnly), + Some(SandboxPolicy::WorkspaceWrite { .. }) => Some(SandboxMode::WorkspaceWrite), + None => None, + }, + network_access: match sandbox_policy { + Some(SandboxPolicy::DangerFullAccess) => Some(NetworkAccess::Enabled), + Some(SandboxPolicy::ReadOnly) => Some(NetworkAccess::Restricted), + Some(SandboxPolicy::WorkspaceWrite { network_access, .. }) => { + if network_access { + Some(NetworkAccess::Enabled) + } else { + Some(NetworkAccess::Restricted) + } + } + None => None, + }, + shell, + } + } +} + +impl EnvironmentContext { + /// Serializes the environment context to XML. Libraries like `quick-xml` + /// require custom macros to handle Enums with newtypes, so we just do it + /// manually, to keep things simple. Output looks like: + /// + /// ```xml + /// + /// ... + /// ... + /// ... + /// ... + /// ... + /// + /// ``` + pub fn serialize_to_xml(self) -> String { + let mut lines = vec![ENVIRONMENT_CONTEXT_START.to_string()]; + if let Some(cwd) = self.cwd { + lines.push(format!(" {}", cwd.to_string_lossy())); + } + if let Some(approval_policy) = self.approval_policy { + lines.push(format!( + " {}", + approval_policy + )); + } + if let Some(sandbox_mode) = self.sandbox_mode { + lines.push(format!(" {}", sandbox_mode)); + } + if let Some(network_access) = self.network_access { + lines.push(format!( + " {}", + network_access + )); + } + if let Some(shell) = self.shell + && let Some(shell_name) = shell.name() + { + lines.push(format!(" {}", shell_name)); + } + lines.push(ENVIRONMENT_CONTEXT_END.to_string()); + lines.join("\n") + } +} + +impl From for ResponseItem { + fn from(ec: EnvironmentContext) -> Self { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: ec.serialize_to_xml(), + }], + } + } +} diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index 2931d30636b..356c23011c2 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -1,8 +1,10 @@ use reqwest::StatusCode; use serde_json; use std::io; +use std::time::Duration; use thiserror::Error; use tokio::task::JoinError; +use uuid::Uuid; pub type Result = std::result::Result; @@ -41,8 +43,16 @@ pub enum CodexErr { /// handshake has succeeded but **before** it finished emitting `response.completed`. /// /// The Session loop treats this as a transient error and will automatically retry the turn. + /// + /// Optionally includes the requested delay before retrying the turn. #[error("stream disconnected before completion: {0}")] - Stream(String), + Stream(String, Option), + + #[error("no conversation with id: {0}")] + ConversationNotFound(Uuid), + + #[error("session configured event was not the first event in the stream")] + SessionConfiguredNotFirstEvent, /// Returned by run_command_stream when the spawned child process timed out (10s). #[error("timeout waiting for child process to exit")] diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index c964466f786..9430433c118 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -3,10 +3,8 @@ use std::os::unix::process::ExitStatusExt; use std::collections::HashMap; use std::io; -use std::path::Path; use std::path::PathBuf; use std::process::ExitStatus; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -15,11 +13,11 @@ use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::BufReader; use tokio::process::Child; -use tokio::sync::Notify; use crate::error::CodexErr; use crate::error::Result; use crate::error::SandboxErr; +use crate::landlock::spawn_command_under_linux_sandbox; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecCommandOutputDeltaEvent; @@ -80,7 +78,6 @@ pub struct StdoutStream { pub async fn process_exec_tool_call( params: ExecParams, sandbox_type: SandboxType, - ctrl_c: Arc, sandbox_policy: &SandboxPolicy, codex_linux_sandbox_exe: &Option, stdout_stream: Option, @@ -89,7 +86,7 @@ pub async fn process_exec_tool_call( let raw_output_result: std::result::Result = match sandbox_type { - SandboxType::None => exec(params, sandbox_policy, ctrl_c, stdout_stream.clone()).await, + SandboxType::None => exec(params, sandbox_policy, stdout_stream.clone()).await, SandboxType::MacosSeatbelt => { let timeout = params.timeout_duration(); let ExecParams { @@ -103,7 +100,7 @@ pub async fn process_exec_tool_call( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream.clone()).await + consume_truncated_output(child, timeout, stdout_stream.clone()).await } SandboxType::LinuxSeccomp => { let timeout = params.timeout_duration(); @@ -124,7 +121,7 @@ pub async fn process_exec_tool_call( ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await + consume_truncated_output(child, timeout, stdout_stream).await } }; let duration = start.elapsed(); @@ -166,65 +163,6 @@ pub async fn process_exec_tool_call( } } -/// Spawn a shell tool command under the Linux Landlock+seccomp sandbox helper -/// (codex-linux-sandbox). -/// -/// Unlike macOS Seatbelt where we directly embed the policy text, the Linux -/// helper accepts a list of `--sandbox-permission`/`-s` flags mirroring the -/// public CLI. We convert the internal [`SandboxPolicy`] representation into -/// the equivalent CLI options. -pub async fn spawn_command_under_linux_sandbox

( - codex_linux_sandbox_exe: P, - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: PathBuf, - stdio_policy: StdioPolicy, - env: HashMap, -) -> std::io::Result -where - P: AsRef, -{ - let args = create_linux_sandbox_command_args(command, sandbox_policy, &cwd); - let arg0 = Some("codex-linux-sandbox"); - spawn_child_async( - codex_linux_sandbox_exe.as_ref().to_path_buf(), - args, - arg0, - cwd, - sandbox_policy, - stdio_policy, - env, - ) - .await -} - -/// Converts the sandbox policy into the CLI invocation for `codex-linux-sandbox`. -fn create_linux_sandbox_command_args( - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: &Path, -) -> Vec { - #[expect(clippy::expect_used)] - let sandbox_policy_cwd = cwd.to_str().expect("cwd must be valid UTF-8").to_string(); - - #[expect(clippy::expect_used)] - let sandbox_policy_json = - serde_json::to_string(sandbox_policy).expect("Failed to serialize SandboxPolicy to JSON"); - - let mut linux_cmd: Vec = vec![ - sandbox_policy_cwd, - sandbox_policy_json, - // Separator so that command arguments starting with `-` are not parsed as - // options of the helper itself. - "--".to_string(), - ]; - - // Append the original tool command. - linux_cmd.extend(command); - - linux_cmd -} - /// We don't have a fully deterministic way to tell if our command failed /// because of the sandbox - a command in the user's zshrc file might hit an /// error, but the command itself might fail or succeed for other reasons. @@ -286,7 +224,6 @@ pub struct ExecToolCallOutput { async fn exec( params: ExecParams, sandbox_policy: &SandboxPolicy, - ctrl_c: Arc, stdout_stream: Option, ) -> Result { let timeout = params.timeout_duration(); @@ -311,14 +248,13 @@ async fn exec( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await + consume_truncated_output(child, timeout, stdout_stream).await } /// Consumes the output of a child process, truncating it so it is suitable for /// use as the output of a `shell` tool call. Also enforces specified timeout. pub(crate) async fn consume_truncated_output( mut child: Child, - ctrl_c: Arc, timeout: Duration, stdout_stream: Option, ) -> Result { @@ -352,7 +288,6 @@ pub(crate) async fn consume_truncated_output( true, )); - let interrupted = ctrl_c.notified(); let exit_status = tokio::select! { result = tokio::time::timeout(timeout, child.wait()) => { match result { @@ -366,7 +301,7 @@ pub(crate) async fn consume_truncated_output( } } } - _ = interrupted => { + _ = tokio::signal::ctrl_c() => { child.start_kill()?; synthetic_exit_status(128 + SIGKILL_CODE) } diff --git a/codex-rs/core/src/exec_env.rs b/codex-rs/core/src/exec_env.rs index 2957f3da15f..88246b063c3 100644 --- a/codex-rs/core/src/exec_env.rs +++ b/codex-rs/core/src/exec_env.rs @@ -70,8 +70,6 @@ where #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used, clippy::expect_used)] - use super::*; use crate::config_types::ShellEnvironmentPolicyInherit; use maplit::hashmap; diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index 52d029f6695..5f25d8fe7b2 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -1,11 +1,16 @@ +use std::collections::HashSet; use std::path::Path; +use codex_protocol::mcp_protocol::GitSha; +use futures::future::join_all; use serde::Deserialize; use serde::Serialize; use tokio::process::Command; use tokio::time::Duration as TokioDuration; use tokio::time::timeout; +use crate::util::is_inside_git_repo; + /// Timeout for git commands to prevent freezing on large repositories const GIT_COMMAND_TIMEOUT: TokioDuration = TokioDuration::from_secs(5); @@ -22,6 +27,12 @@ pub struct GitInfo { pub repository_url: Option, } +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GitDiffToRemote { + pub sha: GitSha, + pub diff: String, +} + /// Collect git repository information from the given working directory using command-line git. /// Returns None if no git repository is found or if git operations fail. /// Uses timeouts to prevent freezing on large repositories. @@ -51,38 +62,52 @@ pub async fn collect_git_info(cwd: &Path) -> Option { }; // Process commit hash - if let Some(output) = commit_result { - if output.status.success() { - if let Ok(hash) = String::from_utf8(output.stdout) { - git_info.commit_hash = Some(hash.trim().to_string()); - } - } + if let Some(output) = commit_result + && output.status.success() + && let Ok(hash) = String::from_utf8(output.stdout) + { + git_info.commit_hash = Some(hash.trim().to_string()); } // Process branch name - if let Some(output) = branch_result { - if output.status.success() { - if let Ok(branch) = String::from_utf8(output.stdout) { - let branch = branch.trim(); - if branch != "HEAD" { - git_info.branch = Some(branch.to_string()); - } - } + if let Some(output) = branch_result + && output.status.success() + && let Ok(branch) = String::from_utf8(output.stdout) + { + let branch = branch.trim(); + if branch != "HEAD" { + git_info.branch = Some(branch.to_string()); } } // Process repository URL - if let Some(output) = url_result { - if output.status.success() { - if let Ok(url) = String::from_utf8(output.stdout) { - git_info.repository_url = Some(url.trim().to_string()); - } - } + if let Some(output) = url_result + && output.status.success() + && let Ok(url) = String::from_utf8(output.stdout) + { + git_info.repository_url = Some(url.trim().to_string()); } Some(git_info) } +/// Returns the closest git sha to HEAD that is on a remote as well as the diff to that sha. +pub async fn git_diff_to_remote(cwd: &Path) -> Option { + if !is_inside_git_repo(cwd) { + return None; + } + + let remotes = get_git_remotes(cwd).await?; + let branches = branch_ancestry(cwd).await?; + let base_sha = find_closest_sha(cwd, &branches, &remotes).await?; + let diff = diff_against_sha(cwd, &base_sha).await?; + + Some(GitDiffToRemote { + sha: base_sha, + diff, + }) +} + /// Run a git command with a timeout to prevent blocking on large repositories async fn run_git_command_with_timeout(args: &[&str], cwd: &Path) -> Option { let result = timeout( @@ -97,11 +122,311 @@ async fn run_git_command_with_timeout(args: &[&str], cwd: &Path) -> Option Option> { + let output = run_git_command_with_timeout(&["remote"], cwd).await?; + if !output.status.success() { + return None; + } + let mut remotes: Vec = String::from_utf8(output.stdout) + .ok()? + .lines() + .map(|s| s.to_string()) + .collect(); + if let Some(pos) = remotes.iter().position(|r| r == "origin") { + let origin = remotes.remove(pos); + remotes.insert(0, origin); + } + Some(remotes) +} + +/// Attempt to determine the repository's default branch name. +/// +/// Preference order: +/// 1) The symbolic ref at `refs/remotes//HEAD` for the first remote (origin prioritized) +/// 2) `git remote show ` parsed for "HEAD branch: " +/// 3) Local fallback to existing `main` or `master` if present +async fn get_default_branch(cwd: &Path) -> Option { + // Prefer the first remote (with origin prioritized) + let remotes = get_git_remotes(cwd).await.unwrap_or_default(); + for remote in remotes { + // Try symbolic-ref, which returns something like: refs/remotes/origin/main + if let Some(symref_output) = run_git_command_with_timeout( + &[ + "symbolic-ref", + "--quiet", + &format!("refs/remotes/{remote}/HEAD"), + ], + cwd, + ) + .await + && symref_output.status.success() + && let Ok(sym) = String::from_utf8(symref_output.stdout) + { + let trimmed = sym.trim(); + if let Some((_, name)) = trimmed.rsplit_once('/') { + return Some(name.to_string()); + } + } + + // Fall back to parsing `git remote show ` output + if let Some(show_output) = + run_git_command_with_timeout(&["remote", "show", &remote], cwd).await + && show_output.status.success() + && let Ok(text) = String::from_utf8(show_output.stdout) + { + for line in text.lines() { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("HEAD branch:") { + let name = rest.trim(); + if !name.is_empty() { + return Some(name.to_string()); + } + } + } + } + } + + // No remote-derived default; try common local defaults if they exist + for candidate in ["main", "master"] { + if let Some(verify) = run_git_command_with_timeout( + &[ + "rev-parse", + "--verify", + "--quiet", + &format!("refs/heads/{candidate}"), + ], + cwd, + ) + .await + && verify.status.success() + { + return Some(candidate.to_string()); + } + } + + None +} + +/// Build an ancestry of branches starting at the current branch and ending at the +/// repository's default branch (if determinable).. +async fn branch_ancestry(cwd: &Path) -> Option> { + // Discover current branch (ignore detached HEAD by treating it as None) + let current_branch = run_git_command_with_timeout(&["rev-parse", "--abbrev-ref", "HEAD"], cwd) + .await + .and_then(|o| { + if o.status.success() { + String::from_utf8(o.stdout).ok() + } else { + None + } + }) + .map(|s| s.trim().to_string()) + .filter(|s| s != "HEAD"); + + // Discover default branch + let default_branch = get_default_branch(cwd).await; + + let mut ancestry: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + if let Some(cb) = current_branch.clone() { + seen.insert(cb.clone()); + ancestry.push(cb); + } + if let Some(db) = default_branch + && !seen.contains(&db) + { + seen.insert(db.clone()); + ancestry.push(db); + } + + // Expand candidates: include any remote branches that already contain HEAD. + // This addresses cases where we're on a new local-only branch forked from a + // remote branch that isn't the repository default. We prioritize remotes in + // the order returned by get_git_remotes (origin first). + let remotes = get_git_remotes(cwd).await.unwrap_or_default(); + for remote in remotes { + if let Some(output) = run_git_command_with_timeout( + &[ + "for-each-ref", + "--format=%(refname:short)", + "--contains=HEAD", + &format!("refs/remotes/{remote}"), + ], + cwd, + ) + .await + && output.status.success() + && let Ok(text) = String::from_utf8(output.stdout) + { + for line in text.lines() { + let short = line.trim(); + // Expect format like: "origin/feature"; extract the branch path after "remote/" + if let Some(stripped) = short.strip_prefix(&format!("{remote}/")) + && !stripped.is_empty() + && !seen.contains(stripped) + { + seen.insert(stripped.to_string()); + ancestry.push(stripped.to_string()); + } + } + } + } + + // Ensure we return Some vector, even if empty, to allow caller logic to proceed + Some(ancestry) +} + +// Helper for a single branch: return the remote SHA if present on any remote +// and the distance (commits ahead of HEAD) for that branch. The first item is +// None if the branch is not present on any remote. Returns None if distance +// could not be computed due to git errors/timeouts. +async fn branch_remote_and_distance( + cwd: &Path, + branch: &str, + remotes: &[String], +) -> Option<(Option, usize)> { + // Try to find the first remote ref that exists for this branch (origin prioritized by caller). + let mut found_remote_sha: Option = None; + let mut found_remote_ref: Option = None; + for remote in remotes { + let remote_ref = format!("refs/remotes/{remote}/{branch}"); + let Some(verify_output) = + run_git_command_with_timeout(&["rev-parse", "--verify", "--quiet", &remote_ref], cwd) + .await + else { + // Mirror previous behavior: if the verify call times out/fails at the process level, + // treat the entire branch as unusable. + return None; + }; + if !verify_output.status.success() { + continue; + } + let Ok(sha) = String::from_utf8(verify_output.stdout) else { + // Mirror previous behavior and skip the entire branch on parse failure. + return None; + }; + found_remote_sha = Some(GitSha::new(sha.trim())); + found_remote_ref = Some(remote_ref); + break; + } + + // Compute distance as the number of commits HEAD is ahead of the branch. + // Prefer local branch name if it exists; otherwise fall back to the remote ref (if any). + let count_output = if let Some(local_count) = + run_git_command_with_timeout(&["rev-list", "--count", &format!("{branch}..HEAD")], cwd) + .await + { + if local_count.status.success() { + local_count + } else if let Some(remote_ref) = &found_remote_ref { + match run_git_command_with_timeout( + &["rev-list", "--count", &format!("{remote_ref}..HEAD")], + cwd, + ) + .await + { + Some(remote_count) => remote_count, + None => return None, + } + } else { + return None; + } + } else if let Some(remote_ref) = &found_remote_ref { + match run_git_command_with_timeout( + &["rev-list", "--count", &format!("{remote_ref}..HEAD")], + cwd, + ) + .await + { + Some(remote_count) => remote_count, + None => return None, + } + } else { + return None; + }; + + if !count_output.status.success() { + return None; + } + let Ok(distance_str) = String::from_utf8(count_output.stdout) else { + return None; + }; + let Ok(distance) = distance_str.trim().parse::() else { + return None; + }; + + Some((found_remote_sha, distance)) +} + +// Finds the closest sha that exist on any of branches and also exists on any of the remotes. +async fn find_closest_sha(cwd: &Path, branches: &[String], remotes: &[String]) -> Option { + // A sha and how many commits away from HEAD it is. + let mut closest_sha: Option<(GitSha, usize)> = None; + for branch in branches { + let Some((maybe_remote_sha, distance)) = + branch_remote_and_distance(cwd, branch, remotes).await + else { + continue; + }; + let Some(remote_sha) = maybe_remote_sha else { + // Preserve existing behavior: skip branches that are not present on a remote. + continue; + }; + match &closest_sha { + None => closest_sha = Some((remote_sha, distance)), + Some((_, best_distance)) if distance < *best_distance => { + closest_sha = Some((remote_sha, distance)); + } + _ => {} + } + } + closest_sha.map(|(sha, _)| sha) +} + +async fn diff_against_sha(cwd: &Path, sha: &GitSha) -> Option { + let output = run_git_command_with_timeout(&["diff", &sha.0], cwd).await?; + // 0 is success and no diff. + // 1 is success but there is a diff. + let exit_ok = output.status.code().is_some_and(|c| c == 0 || c == 1); + if !exit_ok { + return None; + } + let mut diff = String::from_utf8(output.stdout).ok()?; + + if let Some(untracked_output) = + run_git_command_with_timeout(&["ls-files", "--others", "--exclude-standard"], cwd).await + && untracked_output.status.success() + { + let untracked: Vec = String::from_utf8(untracked_output.stdout) + .ok()? + .lines() + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()) + .collect(); + + if !untracked.is_empty() { + let futures_iter = untracked.into_iter().map(|file| async move { + let file_owned = file; + let args_vec: Vec<&str> = + vec!["diff", "--binary", "--no-index", "/dev/null", &file_owned]; + run_git_command_with_timeout(&args_vec, cwd).await + }); + let results = join_all(futures_iter).await; + for extra in results.into_iter().flatten() { + if extra.status.code().is_some_and(|c| c == 0 || c == 1) + && let Ok(s) = String::from_utf8(extra.stdout) + { + diff.push_str(&s); + } + } + } + } + + Some(diff) +} + #[cfg(test)] mod tests { - #![allow(clippy::expect_used)] - #![allow(clippy::unwrap_used)] - use super::*; use std::fs; @@ -110,7 +435,8 @@ mod tests { // Helper function to create a test git repository async fn create_test_git_repo(temp_dir: &TempDir) -> PathBuf { - let repo_path = temp_dir.path().to_path_buf(); + let repo_path = temp_dir.path().join("repo"); + fs::create_dir(&repo_path).expect("Failed to create repo dir"); let envs = vec![ ("GIT_CONFIG_GLOBAL", "/dev/null"), ("GIT_CONFIG_NOSYSTEM", "1"), @@ -165,6 +491,41 @@ mod tests { repo_path } + async fn create_test_git_repo_with_remote(temp_dir: &TempDir) -> (PathBuf, String) { + let repo_path = create_test_git_repo(temp_dir).await; + let remote_path = temp_dir.path().join("remote.git"); + + Command::new("git") + .args(["init", "--bare", remote_path.to_str().unwrap()]) + .output() + .await + .expect("Failed to init bare remote"); + + Command::new("git") + .args(["remote", "add", "origin", remote_path.to_str().unwrap()]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to add remote"); + + let output = Command::new("git") + .args(["rev-parse", "--abbrev-ref", "HEAD"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to get branch"); + let branch = String::from_utf8(output.stdout).unwrap().trim().to_string(); + + Command::new("git") + .args(["push", "-u", "origin", &branch]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to push initial commit"); + + (repo_path, branch) + } + #[tokio::test] async fn test_collect_git_info_non_git_directory() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); @@ -278,6 +639,136 @@ mod tests { assert_eq!(git_info.branch, Some("feature-branch".to_string())); } + #[tokio::test] + async fn test_get_git_working_tree_state_clean_repo() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let (repo_path, branch) = create_test_git_repo_with_remote(&temp_dir).await; + + let remote_sha = Command::new("git") + .args(["rev-parse", &format!("origin/{branch}")]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to rev-parse remote"); + let remote_sha = String::from_utf8(remote_sha.stdout) + .unwrap() + .trim() + .to_string(); + + let state = git_diff_to_remote(&repo_path) + .await + .expect("Should collect working tree state"); + assert_eq!(state.sha, GitSha::new(&remote_sha)); + assert!(state.diff.is_empty()); + } + + #[tokio::test] + async fn test_get_git_working_tree_state_with_changes() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let (repo_path, branch) = create_test_git_repo_with_remote(&temp_dir).await; + + let tracked = repo_path.join("test.txt"); + fs::write(&tracked, "modified").unwrap(); + fs::write(repo_path.join("untracked.txt"), "new").unwrap(); + + let remote_sha = Command::new("git") + .args(["rev-parse", &format!("origin/{branch}")]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to rev-parse remote"); + let remote_sha = String::from_utf8(remote_sha.stdout) + .unwrap() + .trim() + .to_string(); + + let state = git_diff_to_remote(&repo_path) + .await + .expect("Should collect working tree state"); + assert_eq!(state.sha, GitSha::new(&remote_sha)); + assert!(state.diff.contains("test.txt")); + assert!(state.diff.contains("untracked.txt")); + } + + #[tokio::test] + async fn test_get_git_working_tree_state_branch_fallback() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let (repo_path, _branch) = create_test_git_repo_with_remote(&temp_dir).await; + + Command::new("git") + .args(["checkout", "-b", "feature"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to create feature branch"); + Command::new("git") + .args(["push", "-u", "origin", "feature"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to push feature branch"); + + Command::new("git") + .args(["checkout", "-b", "local-branch"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to create local branch"); + + let remote_sha = Command::new("git") + .args(["rev-parse", "origin/feature"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to rev-parse remote"); + let remote_sha = String::from_utf8(remote_sha.stdout) + .unwrap() + .trim() + .to_string(); + + let state = git_diff_to_remote(&repo_path) + .await + .expect("Should collect working tree state"); + assert_eq!(state.sha, GitSha::new(&remote_sha)); + } + + #[tokio::test] + async fn test_get_git_working_tree_state_unpushed_commit() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let (repo_path, branch) = create_test_git_repo_with_remote(&temp_dir).await; + + let remote_sha = Command::new("git") + .args(["rev-parse", &format!("origin/{branch}")]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to rev-parse remote"); + let remote_sha = String::from_utf8(remote_sha.stdout) + .unwrap() + .trim() + .to_string(); + + fs::write(repo_path.join("test.txt"), "updated").unwrap(); + Command::new("git") + .args(["add", "test.txt"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to add file"); + Command::new("git") + .args(["commit", "-m", "local change"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to commit"); + + let state = git_diff_to_remote(&repo_path) + .await + .expect("Should collect working tree state"); + assert_eq!(state.sha, GitSha::new(&remote_sha)); + assert!(state.diff.contains("updated")); + } + #[test] fn test_git_info_serialization() { let git_info = GitInfo { diff --git a/codex-rs/core/src/is_safe_command.rs b/codex-rs/core/src/is_safe_command.rs index f5f453f8d85..f54e247fd86 100644 --- a/codex-rs/core/src/is_safe_command.rs +++ b/codex-rs/core/src/is_safe_command.rs @@ -12,20 +12,17 @@ pub fn is_known_safe_command(command: &[String]) -> bool { // introduce side effects ( "&&", "||", ";", and "|" ). If every // individual command in the script is itself a known‑safe command, then // the composite expression is considered safe. - if let [bash, flag, script] = command { - if bash == "bash" && flag == "-lc" { - if let Some(tree) = try_parse_bash(script) { - if let Some(all_commands) = try_parse_word_only_commands_sequence(&tree, script) { - if !all_commands.is_empty() - && all_commands - .iter() - .all(|cmd| is_safe_to_call_with_exec(cmd)) - { - return true; - } - } - } - } + if let [bash, flag, script] = command + && bash == "bash" + && flag == "-lc" + && let Some(tree) = try_parse_bash(script) + && let Some(all_commands) = try_parse_word_only_commands_sequence(&tree, script) + && !all_commands.is_empty() + && all_commands + .iter() + .all(|cmd| is_safe_to_call_with_exec(cmd)) + { + return true; } false @@ -162,7 +159,6 @@ fn is_valid_sed_n_arg(arg: Option<&str>) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; fn vec_str(args: &[&str]) -> Vec { diff --git a/codex-rs/core/src/landlock.rs b/codex-rs/core/src/landlock.rs new file mode 100644 index 00000000000..8c8840c2695 --- /dev/null +++ b/codex-rs/core/src/landlock.rs @@ -0,0 +1,66 @@ +use crate::protocol::SandboxPolicy; +use crate::spawn::StdioPolicy; +use crate::spawn::spawn_child_async; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tokio::process::Child; + +/// Spawn a shell tool command under the Linux Landlock+seccomp sandbox helper +/// (codex-linux-sandbox). +/// +/// Unlike macOS Seatbelt where we directly embed the policy text, the Linux +/// helper accepts a list of `--sandbox-permission`/`-s` flags mirroring the +/// public CLI. We convert the internal [`SandboxPolicy`] representation into +/// the equivalent CLI options. +pub async fn spawn_command_under_linux_sandbox

( + codex_linux_sandbox_exe: P, + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result +where + P: AsRef, +{ + let args = create_linux_sandbox_command_args(command, sandbox_policy, &cwd); + let arg0 = Some("codex-linux-sandbox"); + spawn_child_async( + codex_linux_sandbox_exe.as_ref().to_path_buf(), + args, + arg0, + cwd, + sandbox_policy, + stdio_policy, + env, + ) + .await +} + +/// Converts the sandbox policy into the CLI invocation for `codex-linux-sandbox`. +fn create_linux_sandbox_command_args( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: &Path, +) -> Vec { + #[expect(clippy::expect_used)] + let sandbox_policy_cwd = cwd.to_str().expect("cwd must be valid UTF-8").to_string(); + + #[expect(clippy::expect_used)] + let sandbox_policy_json = + serde_json::to_string(sandbox_policy).expect("Failed to serialize SandboxPolicy to JSON"); + + let mut linux_cmd: Vec = vec![ + sandbox_policy_cwd, + sandbox_policy_json, + // Separator so that command arguments starting with `-` are not parsed as + // options of the helper itself. + "--".to_string(), + ]; + + // Append the original tool command. + linux_cmd.extend(command); + + linux_cmd +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index f3247c38873..40ccbf67698 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -11,19 +11,20 @@ mod chat_completions; mod client; mod client_common; pub mod codex; -pub use codex::Codex; -pub use codex::CodexSpawnOk; -pub mod codex_wrapper; +mod codex_conversation; +pub use codex_conversation::CodexConversation; pub mod config; pub mod config_profile; pub mod config_types; mod conversation_history; +mod environment_context; pub mod error; pub mod exec; pub mod exec_env; mod flags; pub mod git_info; mod is_safe_command; +pub mod landlock; mod mcp_connection_manager; mod mcp_tool_call; mod message_history; @@ -34,21 +35,30 @@ pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; pub use model_provider_info::built_in_model_providers; pub use model_provider_info::create_oss_provider_with_base_url; +mod conversation_manager; +pub use conversation_manager::ConversationManager; +pub use conversation_manager::NewConversation; pub mod model_family; mod models; mod openai_model_info; mod openai_tools; pub mod plan_tool; -mod project_doc; -pub mod protocol; +pub mod project_doc; mod rollout; pub(crate) mod safety; pub mod seatbelt; pub mod shell; pub mod spawn; +pub mod terminal; pub mod turn_diff_tracker; pub mod user_agent; mod user_notification; pub mod util; pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use safety::get_platform_sandbox; +// Re-export the protocol types from the standalone `codex-protocol` crate so existing +// `codex_core::protocol::...` references continue to work across the workspace. +pub use codex_protocol::protocol; +// Re-export protocol config enums to ensure call sites can use the same types +// as those in the protocol crate when constructing protocol messages. +pub use codex_protocol::config_types as protocol_config_types; diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index 2e33c8754b9..b5813a04627 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -281,7 +281,6 @@ fn is_valid_mcp_server_name(server_name: &str) -> bool { } #[cfg(test)] -#[allow(clippy::unwrap_used)] mod tests { use super::*; use mcp_types::ToolInputSchema; diff --git a/codex-rs/core/src/message_history.rs b/codex-rs/core/src/message_history.rs index 29970c2a74e..9f13ababb73 100644 --- a/codex-rs/core/src/message_history.rs +++ b/codex-rs/core/src/message_history.rs @@ -125,16 +125,18 @@ pub(crate) async fn append_entry(text: &str, session_id: &Uuid, config: &Config) /// times if the lock is currently held by another process. This prevents a /// potential indefinite wait while still giving other writers some time to /// finish their operation. -async fn acquire_exclusive_lock_with_retry(file: &std::fs::File) -> Result<()> { +async fn acquire_exclusive_lock_with_retry(file: &File) -> Result<()> { use tokio::time::sleep; for _ in 0..MAX_RETRIES { - match fs2::FileExt::try_lock_exclusive(file) { + match file.try_lock() { Ok(()) => return Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { - sleep(RETRY_SLEEP).await; - } - Err(e) => return Err(e), + Err(e) => match e { + std::fs::TryLockError::WouldBlock => { + sleep(RETRY_SLEEP).await; + } + other => return Err(other.into()), + }, } } @@ -259,12 +261,14 @@ pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option Result<()> { for _ in 0..MAX_RETRIES { - match fs2::FileExt::try_lock_shared(file) { + match file.try_lock_shared() { Ok(()) => return Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(RETRY_SLEEP); - } - Err(e) => return Err(e), + Err(e) => match e { + std::fs::TryLockError::WouldBlock => { + std::thread::sleep(RETRY_SLEEP); + } + other => return Err(other.into()), + }, } } diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index 1245a030c66..6d1c2efcc1f 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -23,6 +23,10 @@ pub struct ModelFamily { // the model such that its description can be omitted. // See https://platform.openai.com/docs/guides/tools-local-shell pub uses_local_shell_tool: bool, + + /// True if the model performs better when `apply_patch` is provided as + /// a tool call instead of just a bash command. + pub uses_apply_patch_tool: bool, } macro_rules! model_family { @@ -36,6 +40,7 @@ macro_rules! model_family { needs_special_apply_patch_instructions: false, supports_reasoning_summaries: false, uses_local_shell_tool: false, + uses_apply_patch_tool: false, }; // apply overrides $( @@ -55,6 +60,7 @@ macro_rules! simple_model_family { needs_special_apply_patch_instructions: false, supports_reasoning_summaries: false, uses_local_shell_tool: false, + uses_apply_patch_tool: false, }) }}; } @@ -78,15 +84,20 @@ pub fn find_family_for_model(slug: &str) -> Option { supports_reasoning_summaries: true, uses_local_shell_tool: true, ) + } else if slug.starts_with("codex-") { + model_family!( + slug, slug, + supports_reasoning_summaries: true, + ) } else if slug.starts_with("gpt-4.1") { model_family!( slug, "gpt-4.1", needs_special_apply_patch_instructions: true, ) + } else if slug.starts_with("gpt-oss") { + model_family!(slug, "gpt-oss", uses_apply_patch_tool: true) } else if slug.starts_with("gpt-4o") { simple_model_family!(slug, "gpt-4o") - } else if slug.starts_with("gpt-oss") { - simple_model_family!(slug, "gpt-oss") } else if slug.starts_with("gpt-3.5") { simple_model_family!(slug, "gpt-3.5") } else if slug.starts_with("gpt-5") { diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 98f07deb1eb..0102591d2c1 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -167,10 +167,10 @@ impl ModelProviderInfo { if let Some(env_headers) = &self.env_http_headers { for (header, env_var) in env_headers { - if let Ok(val) = std::env::var(env_var) { - if !val.trim().is_empty() { - builder = builder.header(header, val); - } + if let Ok(val) = std::env::var(env_var) + && !val.trim().is_empty() + { + builder = builder.header(header, val); } } } @@ -322,7 +322,6 @@ pub fn create_oss_provider_with_base_url(base_url: &str) -> ModelProviderInfo { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; use pretty_assertions::assert_eq; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index e052bc43a41..f6323e2724d 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -45,7 +45,7 @@ pub enum ResponseItem { Reasoning { id: String, summary: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "should_serialize_reasoning_content")] content: Option>, encrypted_content: Option, }, @@ -81,6 +81,15 @@ pub enum ResponseItem { Other, } +fn should_serialize_reasoning_content(content: &Option>) -> bool { + match content { + Some(content) => !content + .iter() + .any(|c| matches!(c, ReasoningItemContent::ReasoningText { .. })), + None => false, + } +} + impl From for ResponseItem { fn from(item: ResponseInputItem) -> Self { match item { @@ -142,6 +151,7 @@ pub enum ReasoningItemReasoningSummary { #[serde(tag = "type", rename_all = "snake_case")] pub enum ReasoningItemContent { ReasoningText { text: String }, + Text { text: String }, } impl From> for ResponseInputItem { @@ -173,6 +183,7 @@ impl From> for ResponseInputItem { None } }, + _ => None, }) .collect::>(), } @@ -256,7 +267,6 @@ impl std::ops::Deref for FunctionCallOutputPayload { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; #[test] diff --git a/codex-rs/core/src/openai_model_info.rs b/codex-rs/core/src/openai_model_info.rs index a072d409c60..66f3c626ea1 100644 --- a/codex-rs/core/src/openai_model_info.rs +++ b/codex-rs/core/src/openai_model_info.rs @@ -15,7 +15,8 @@ pub(crate) struct ModelInfo { } pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { - match model_family.slug.as_str() { + let slug = model_family.slug.as_str(); + match slug { // OSS models have a 128k shared token pool. // Arbitrarily splitting it: 3/4 input context, 1/4 output. // https://openai.com/index/gpt-oss-model-card/ @@ -82,6 +83,11 @@ pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { max_output_tokens: 100_000, }), + _ if slug.starts_with("codex-") => Some(ModelInfo { + context_window: 200_000, + max_output_tokens: 100_000, + }), + _ => None, } } diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index ad794c38f1d..dd5eb125161 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -43,6 +43,7 @@ pub enum ConfigShellToolType { pub struct ToolsConfig { pub shell_type: ConfigShellToolType, pub plan_tool: bool, + pub apply_patch_tool: bool, } impl ToolsConfig { @@ -51,6 +52,7 @@ impl ToolsConfig { approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, include_plan_tool: bool, + include_apply_patch_tool: bool, ) -> Self { let mut shell_type = if model_family.uses_local_shell_tool { ConfigShellToolType::LocalShell @@ -66,6 +68,7 @@ impl ToolsConfig { Self { shell_type, plan_tool: include_plan_tool, + apply_patch_tool: include_apply_patch_tool || model_family.uses_apply_patch_tool, } } } @@ -235,6 +238,87 @@ The shell tool is used to execute shell commands. }) } +#[derive(Serialize, Deserialize)] +pub(crate) struct ApplyPatchToolArgs { + pub(crate) input: String, +} + +fn create_apply_patch_tool() -> OpenAiTool { + // Minimal schema: one required string argument containing the patch body + let mut properties = BTreeMap::new(); + properties.insert( + "input".to_string(), + JsonSchema::String { + description: Some(r#"The entire contents of the apply_patch command"#.to_string()), + }, + ); + + OpenAiTool::Function(ResponsesApiTool { + name: "apply_patch".to_string(), + description: r#"Use this tool to edit files. +Your patch language is a stripped‑down, file‑oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high‑level envelope: + +**_ Begin Patch +[ one or more file sections ] +_** End Patch + +Within that envelope, you get a sequence of file operations. +You MUST include a header to specify the action you are taking. +Each operation starts with one of three headers: + +**_ Add File: - create a new file. Every following line is a + line (the initial contents). +_** Delete File: - remove an existing file. Nothing follows. +\*\*\* Update File: - patch an existing file in place (optionally with a rename). + +May be immediately followed by \*\*\* Move to: if you want to rename the file. +Then one or more “hunks”, each introduced by @@ (optionally followed by a hunk header). +Within a hunk each line starts with: + +- for inserted text, + +* for removed text, or + space ( ) for context. + At the end of a truncated hunk you can emit \*\*\* End of File. + +Patch := Begin { FileOp } End +Begin := "**_ Begin Patch" NEWLINE +End := "_** End Patch" NEWLINE +FileOp := AddFile | DeleteFile | UpdateFile +AddFile := "**_ Add File: " path NEWLINE { "+" line NEWLINE } +DeleteFile := "_** Delete File: " path NEWLINE +UpdateFile := "**_ Update File: " path NEWLINE [ MoveTo ] { Hunk } +MoveTo := "_** Move to: " newPath NEWLINE +Hunk := "@@" [ header ] NEWLINE { HunkLine } [ "*** End of File" NEWLINE ] +HunkLine := (" " | "-" | "+") text NEWLINE + +A full patch can combine several operations: + +**_ Begin Patch +_** Add File: hello.txt ++Hello world +**_ Update File: src/app.py +_** Move to: src/main.py +@@ def greet(): +-print("Hi") ++print("Hello, world!") +**_ Delete File: obsolete.txt +_** End Patch + +It is important to remember: + +- You must include a header with your intended action (Add/Delete/Update) +- You must prefix new lines with `+` even when creating a new file +"# + .to_string(), + strict: false, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["input".to_string()]), + additional_properties: Some(false), + }, + }) +} + /// Returns JSON values that are compatible with Function Calling in the /// Responses API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=responses @@ -336,11 +420,11 @@ fn sanitize_json_schema(value: &mut JsonValue) { } JsonValue::Object(map) => { // First, recursively sanitize known nested schema holders - if let Some(props) = map.get_mut("properties") { - if let Some(props_map) = props.as_object_mut() { - for (_k, v) in props_map.iter_mut() { - sanitize_json_schema(v); - } + if let Some(props) = map.get_mut("properties") + && let Some(props_map) = props.as_object_mut() + { + for (_k, v) in props_map.iter_mut() { + sanitize_json_schema(v); } } if let Some(items) = map.get_mut("items") { @@ -360,18 +444,18 @@ fn sanitize_json_schema(value: &mut JsonValue) { .map(|s| s.to_string()); // If type is an array (union), pick first supported; else leave to inference - if ty.is_none() { - if let Some(JsonValue::Array(types)) = map.get("type") { - for t in types { - if let Some(tt) = t.as_str() { - if matches!( - tt, - "object" | "array" | "string" | "number" | "integer" | "boolean" - ) { - ty = Some(tt.to_string()); - break; - } - } + if ty.is_none() + && let Some(JsonValue::Array(types)) = map.get("type") + { + for t in types { + if let Some(tt) = t.as_str() + && matches!( + tt, + "object" | "array" | "string" | "number" | "integer" | "boolean" + ) + { + ty = Some(tt.to_string()); + break; } } } @@ -455,6 +539,10 @@ pub(crate) fn get_openai_tools( tools.push(PLAN_TOOL.clone()); } + if config.apply_patch_tool { + tools.push(create_apply_patch_tool()); + } + if let Some(mcp_tools) = mcp_tools { for (name, tool) in mcp_tools { match mcp_tool_to_openai_tool(name.clone(), tool.clone()) { @@ -470,7 +558,6 @@ pub(crate) fn get_openai_tools( } #[cfg(test)] -#[allow(clippy::expect_used)] mod tests { use crate::model_family::find_family_for_model; use mcp_types::ToolInputSchema; @@ -509,6 +596,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, true, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools(&config, Some(HashMap::new())); @@ -523,6 +611,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, true, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools(&config, Some(HashMap::new())); @@ -537,6 +626,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( &config, @@ -630,6 +720,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( @@ -685,6 +776,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( @@ -735,6 +827,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( @@ -788,6 +881,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( diff --git a/codex-rs/core/src/parse_command.rs b/codex-rs/core/src/parse_command.rs index 01dc6e32279..7348af43ac2 100644 --- a/codex-rs/core/src/parse_command.rs +++ b/codex-rs/core/src/parse_command.rs @@ -3,40 +3,67 @@ use crate::bash::try_parse_word_only_commands_sequence; use serde::Deserialize; use serde::Serialize; use shlex::split as shlex_split; +use shlex::try_join as shlex_try_join; #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub enum ParsedCommand { Read { - cmd: Vec, + cmd: String, name: String, }, ListFiles { - cmd: Vec, + cmd: String, path: Option, }, Search { - cmd: Vec, + cmd: String, query: Option, path: Option, }, Format { - cmd: Vec, + cmd: String, tool: Option, targets: Option>, }, Test { - cmd: Vec, + cmd: String, }, Lint { - cmd: Vec, + cmd: String, tool: Option, targets: Option>, }, + Noop { + cmd: String, + }, Unknown { - cmd: Vec, + cmd: String, }, } +// Convert core's parsed command enum into the protocol's simplified type so +// events can carry the canonical representation across process boundaries. +impl From for codex_protocol::parse_command::ParsedCommand { + fn from(v: ParsedCommand) -> Self { + use codex_protocol::parse_command::ParsedCommand as P; + match v { + ParsedCommand::Read { cmd, name } => P::Read { cmd, name }, + ParsedCommand::ListFiles { cmd, path } => P::ListFiles { cmd, path }, + ParsedCommand::Search { cmd, query, path } => P::Search { cmd, query, path }, + ParsedCommand::Format { cmd, tool, targets } => P::Format { cmd, tool, targets }, + ParsedCommand::Test { cmd } => P::Test { cmd }, + ParsedCommand::Lint { cmd, tool, targets } => P::Lint { cmd, tool, targets }, + ParsedCommand::Noop { cmd } => P::Noop { cmd }, + ParsedCommand::Unknown { cmd } => P::Unknown { cmd }, + } + } +} + +fn shlex_join(tokens: &[String]) -> String { + shlex_try_join(tokens.iter().map(|s| s.as_str())) + .unwrap_or_else(|_| "".to_string()) +} + /// DO NOT REVIEW THIS CODE BY HAND /// This parsing code is quite complex and not easy to hand-modify. /// The easiest way to iterate is to add unit tests and have Codex fix the implementation. @@ -84,7 +111,29 @@ mod tests { assert_parsed( &vec_str(&["git", "status"]), vec![ParsedCommand::Unknown { - cmd: vec_str(&["git", "status"]), + cmd: "git status".to_string(), + }], + ); + } + + #[test] + fn handles_git_pipe_wc() { + let inner = "git status | wc -l"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Unknown { + cmd: "git status | wc -l".to_string(), + }], + ); + } + + #[test] + fn bash_lc_redirect_not_quoted() { + let inner = "echo foo > bar"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Unknown { + cmd: "echo foo > bar".to_string(), }], ); } @@ -98,23 +147,23 @@ mod tests { vec![ // Expect commands in left-to-right execution order ParsedCommand::Search { - cmd: vec_str(&["rg", "--version"]), + cmd: "rg --version".to_string(), query: None, path: None, }, ParsedCommand::Unknown { - cmd: vec_str(&["node", "-v"]), + cmd: "node -v".to_string(), }, ParsedCommand::Unknown { - cmd: vec_str(&["pnpm", "-v"]), + cmd: "pnpm -v".to_string(), }, ParsedCommand::Search { - cmd: vec_str(&["rg", "--files"]), + cmd: "rg --files".to_string(), query: None, path: None, }, ParsedCommand::Unknown { - cmd: vec_str(&["head", "-n", "40"]), + cmd: "head -n 40".to_string(), }, ], ); @@ -126,7 +175,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Search { - cmd: shlex_split_safe(inner), + cmd: "rg -n navigate-to-route -S".to_string(), query: Some("navigate-to-route".to_string()), path: None, }], @@ -141,12 +190,12 @@ mod tests { &vec_str(&["bash", "-lc", inner]), vec![ ParsedCommand::Search { - cmd: vec_str(&["rg", "-n", "BUG|FIXME|TODO|XXX|HACK", "-S"]), + cmd: "rg -n 'BUG|FIXME|TODO|XXX|HACK' -S".to_string(), query: Some("BUG|FIXME|TODO|XXX|HACK".to_string()), path: None, }, ParsedCommand::Unknown { - cmd: vec_str(&["head", "-n", "200"]), + cmd: "head -n 200".to_string(), }, ], ); @@ -158,7 +207,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Search { - cmd: vec_str(&["rg", "--files", "webview/src"]), + cmd: "rg --files webview/src".to_string(), query: None, path: Some("webview".to_string()), }], @@ -172,12 +221,12 @@ mod tests { &vec_str(&["bash", "-lc", inner]), vec![ ParsedCommand::Search { - cmd: vec_str(&["rg", "--files"]), + cmd: "rg --files".to_string(), query: None, path: None, }, ParsedCommand::Unknown { - cmd: vec_str(&["head", "-n", "50"]), + cmd: "head -n 50".to_string(), }, ], ); @@ -189,7 +238,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "README.md".to_string(), }], ); @@ -201,7 +250,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::ListFiles { - cmd: vec_str(&["ls", "-la"]), + cmd: "ls -la".to_string(), path: None, }], ); @@ -213,7 +262,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "Cargo.toml".to_string(), }], ); @@ -225,7 +274,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "Cargo.toml".to_string(), }], ); @@ -237,7 +286,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "README.md".to_string(), }], ); @@ -250,7 +299,7 @@ mod tests { assert_eq!( out, vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "README.md".to_string(), }] ); @@ -261,7 +310,7 @@ mod tests { assert_parsed( &vec_str(&["npm", "run", "build"]), vec![ParsedCommand::Unknown { - cmd: vec_str(&["npm", "run", "build"]), + cmd: "npm run build".to_string(), }], ); } @@ -280,16 +329,7 @@ mod tests { "json", ]), vec![ParsedCommand::Lint { - cmd: vec_str(&[ - "npm", - "run", - "lint", - "--", - "--max-warnings", - "0", - "--format", - "json", - ]), + cmd: "npm run lint -- --max-warnings 0 --format json".to_string(), tool: Some("npm-script:lint".to_string()), targets: None, }], @@ -301,7 +341,7 @@ mod tests { assert_parsed( &vec_str(&["grep", "-R", "CODEX_SANDBOX_ENV_VAR", "-n", "."]), vec![ParsedCommand::Search { - cmd: vec_str(&["grep", "-R", "CODEX_SANDBOX_ENV_VAR", "-n", "."]), + cmd: "grep -R CODEX_SANDBOX_ENV_VAR -n .".to_string(), query: Some("CODEX_SANDBOX_ENV_VAR".to_string()), path: Some(".".to_string()), }], @@ -319,13 +359,7 @@ mod tests { "core/src/spawn.rs", ]), vec![ParsedCommand::Search { - cmd: vec_str(&[ - "grep", - "-R", - "CODEX_SANDBOX_ENV_VAR", - "-n", - "core/src/spawn.rs", - ]), + cmd: "grep -R CODEX_SANDBOX_ENV_VAR -n core/src/spawn.rs".to_string(), query: Some("CODEX_SANDBOX_ENV_VAR".to_string()), path: Some("spawn.rs".to_string()), }], @@ -339,7 +373,7 @@ mod tests { assert_parsed( &shlex_split_safe("grep -R src/main.rs -n ."), vec![ParsedCommand::Search { - cmd: vec_str(&["grep", "-R", "src/main.rs", "-n", "."]), + cmd: "grep -R src/main.rs -n .".to_string(), query: Some("src/main.rs".to_string()), path: Some(".".to_string()), }], @@ -351,7 +385,7 @@ mod tests { assert_parsed( &shlex_split_safe("grep -R COD`EX_SANDBOX -n"), vec![ParsedCommand::Search { - cmd: vec_str(&["grep", "-R", "COD`EX_SANDBOX", "-n"]), + cmd: "grep -R 'COD`EX_SANDBOX' -n".to_string(), query: Some("COD`EX_SANDBOX".to_string()), path: None, }], @@ -364,10 +398,10 @@ mod tests { &shlex_split_safe("cd codex-rs && rg --files"), vec![ ParsedCommand::Unknown { - cmd: vec_str(&["cd", "codex-rs"]), + cmd: "cd codex-rs".to_string(), }, ParsedCommand::Search { - cmd: vec_str(&["rg", "--files"]), + cmd: "rg --files".to_string(), query: None, path: None, }, @@ -380,7 +414,7 @@ mod tests { assert_parsed( &shlex_split_safe("echo Running tests... && cargo test --all-features --quiet"), vec![ParsedCommand::Test { - cmd: vec_str(&["cargo", "test", "--all-features", "--quiet"]), + cmd: "cargo test --all-features --quiet".to_string(), }], ); } @@ -393,12 +427,12 @@ mod tests { ), vec![ ParsedCommand::Format { - cmd: shlex_split_safe("cargo fmt -- --config imports_granularity=Item"), + cmd: "cargo fmt -- --config 'imports_granularity=Item'".to_string(), tool: Some("cargo fmt".to_string()), targets: None, }, ParsedCommand::Test { - cmd: vec_str(&["cargo", "test", "-p", "core", "--all-features"]), + cmd: "cargo test -p core --all-features".to_string(), }, ], ); @@ -409,7 +443,7 @@ mod tests { assert_parsed( &shlex_split_safe("rustfmt src/main.rs"), vec![ParsedCommand::Format { - cmd: vec_str(&["rustfmt", "src/main.rs"]), + cmd: "rustfmt src/main.rs".to_string(), tool: Some("rustfmt".to_string()), targets: Some(vec!["src/main.rs".to_string()]), }], @@ -418,16 +452,7 @@ mod tests { assert_parsed( &shlex_split_safe("cargo clippy -p core --all-features -- -D warnings"), vec![ParsedCommand::Lint { - cmd: vec_str(&[ - "cargo", - "clippy", - "-p", - "core", - "--all-features", - "--", - "-D", - "warnings", - ]), + cmd: "cargo clippy -p core --all-features -- -D warnings".to_string(), tool: Some("cargo clippy".to_string()), targets: None, }], @@ -441,19 +466,15 @@ mod tests { "pytest -k 'Login and not slow' tests/test_login.py::TestLogin::test_ok", ), vec![ParsedCommand::Test { - cmd: vec_str(&[ - "pytest", - "-k", - "Login and not slow", - "tests/test_login.py::TestLogin::test_ok", - ]), + cmd: "pytest -k 'Login and not slow' tests/test_login.py::TestLogin::test_ok" + .to_string(), }], ); assert_parsed( &shlex_split_safe("go fmt ./..."), vec![ParsedCommand::Format { - cmd: vec_str(&["go", "fmt", "./..."]), + cmd: "go fmt ./...".to_string(), tool: Some("go fmt".to_string()), targets: Some(vec!["./...".to_string()]), }], @@ -462,14 +483,14 @@ mod tests { assert_parsed( &shlex_split_safe("go test ./pkg -run TestThing"), vec![ParsedCommand::Test { - cmd: vec_str(&["go", "test", "./pkg", "-run", "TestThing"]), + cmd: "go test ./pkg -run TestThing".to_string(), }], ); assert_parsed( &shlex_split_safe("eslint . --max-warnings 0"), vec![ParsedCommand::Lint { - cmd: vec_str(&["eslint", ".", "--max-warnings", "0"]), + cmd: "eslint . --max-warnings 0".to_string(), tool: Some("eslint".to_string()), targets: Some(vec![".".to_string()]), }], @@ -478,7 +499,7 @@ mod tests { assert_parsed( &shlex_split_safe("prettier -w ."), vec![ParsedCommand::Format { - cmd: vec_str(&["prettier", "-w", "."]), + cmd: "prettier -w .".to_string(), tool: Some("prettier".to_string()), targets: Some(vec![".".to_string()]), }], @@ -490,14 +511,14 @@ mod tests { assert_parsed( &shlex_split_safe("jest -t 'should work' src/foo.test.ts"), vec![ParsedCommand::Test { - cmd: vec_str(&["jest", "-t", "should work", "src/foo.test.ts"]), + cmd: "jest -t 'should work' src/foo.test.ts".to_string(), }], ); assert_parsed( &shlex_split_safe("vitest -t 'runs' src/foo.test.tsx"), vec![ParsedCommand::Test { - cmd: vec_str(&["vitest", "-t", "runs", "src/foo.test.tsx"]), + cmd: "vitest -t runs src/foo.test.tsx".to_string(), }], ); } @@ -507,7 +528,7 @@ mod tests { assert_parsed( &shlex_split_safe("npx eslint src"), vec![ParsedCommand::Lint { - cmd: vec_str(&["npx", "eslint", "src"]), + cmd: "npx eslint src".to_string(), tool: Some("eslint".to_string()), targets: Some(vec!["src".to_string()]), }], @@ -516,7 +537,7 @@ mod tests { assert_parsed( &shlex_split_safe("npx prettier -c ."), vec![ParsedCommand::Format { - cmd: vec_str(&["npx", "prettier", "-c", "."]), + cmd: "npx prettier -c .".to_string(), tool: Some("prettier".to_string()), targets: Some(vec![".".to_string()]), }], @@ -525,7 +546,7 @@ mod tests { assert_parsed( &shlex_split_safe("pnpm run lint -- --max-warnings 0"), vec![ParsedCommand::Lint { - cmd: vec_str(&["pnpm", "run", "lint", "--", "--max-warnings", "0"]), + cmd: "pnpm run lint -- --max-warnings 0".to_string(), tool: Some("pnpm-script:lint".to_string()), targets: None, }], @@ -534,14 +555,14 @@ mod tests { assert_parsed( &shlex_split_safe("npm test"), vec![ParsedCommand::Test { - cmd: vec_str(&["npm", "test"]), + cmd: "npm test".to_string(), }], ); assert_parsed( &shlex_split_safe("yarn test"), vec![ParsedCommand::Test { - cmd: vec_str(&["yarn", "test"]), + cmd: "yarn test".to_string(), }], ); } @@ -631,7 +652,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "parse_command.rs".to_string(), }], ); @@ -643,7 +664,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe(inner), + cmd: inner.to_string(), name: "history_cell.rs".to_string(), }], ); @@ -656,7 +677,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Read { - cmd: shlex_split_safe("cat -- ansi-escape/Cargo.toml"), + cmd: "cat -- ansi-escape/Cargo.toml".to_string(), name: "Cargo.toml".to_string(), }], ); @@ -669,7 +690,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Search { - cmd: vec_str(&["rg", "--files"]), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -685,9 +706,7 @@ mod tests { assert_parsed( &args, vec![ParsedCommand::Read { - cmd: shlex_split_safe( - "sed -n '260,640p' exec/src/event_processor_with_human_output.rs", - ), + cmd: "sed -n '260,640p' exec/src/event_processor_with_human_output.rs".to_string(), name: "event_processor_with_human_output.rs".to_string(), }], ); @@ -698,7 +717,7 @@ mod tests { assert_parsed( &shlex_split_safe("yes | rg -n 'foo bar' -S"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg -n 'foo bar' -S"), + cmd: "rg -n 'foo bar' -S".to_string(), query: Some("foo bar".to_string()), path: None, }], @@ -710,7 +729,7 @@ mod tests { assert_parsed( &shlex_split_safe("ls -I '*.test.js'"), vec![ParsedCommand::ListFiles { - cmd: shlex_split_safe("ls -I '*.test.js'"), + cmd: "ls -I '*.test.js'".to_string(), path: None, }], ); @@ -722,12 +741,12 @@ mod tests { &shlex_split_safe("rg foo ; echo done"), vec![ ParsedCommand::Search { - cmd: shlex_split_safe("rg foo"), + cmd: "rg foo".to_string(), query: Some("foo".to_string()), path: None, }, ParsedCommand::Unknown { - cmd: shlex_split_safe("echo done"), + cmd: "echo done".to_string(), }, ], ); @@ -740,12 +759,12 @@ mod tests { &shlex_split_safe("rg foo || echo done"), vec![ ParsedCommand::Search { - cmd: shlex_split_safe("rg foo"), + cmd: "rg foo".to_string(), query: Some("foo".to_string()), path: None, }, ParsedCommand::Unknown { - cmd: shlex_split_safe("echo done"), + cmd: "echo done".to_string(), }, ], ); @@ -757,7 +776,7 @@ mod tests { assert_parsed( &shlex_split_safe("true && rg --files"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -766,7 +785,7 @@ mod tests { assert_parsed( &shlex_split_safe("rg --files && true"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -779,7 +798,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner]), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -789,7 +808,7 @@ mod tests { assert_parsed( &vec_str(&["bash", "-lc", inner2]), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -801,7 +820,7 @@ mod tests { assert_parsed( &shlex_split_safe(r#"cat "pkg\src\main.rs""#), vec![ParsedCommand::Read { - cmd: shlex_split_safe(r#"cat "pkg\src\main.rs""#), + cmd: r#"cat "pkg\\src\\main.rs""#.to_string(), name: "main.rs".to_string(), }], ); @@ -812,7 +831,7 @@ mod tests { assert_parsed( &shlex_split_safe("bash -lc 'head -n50 Cargo.toml'"), vec![ParsedCommand::Read { - cmd: shlex_split_safe("head -n50 Cargo.toml"), + cmd: "head -n50 Cargo.toml".to_string(), name: "Cargo.toml".to_string(), }], ); @@ -826,12 +845,12 @@ mod tests { &shlex_split_safe(inner), vec![ ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }, ParsedCommand::Unknown { - cmd: shlex_split_safe("head -n 1"), + cmd: "head -n 1".to_string(), }, ], ); @@ -842,7 +861,7 @@ mod tests { assert_parsed( &shlex_split_safe("bash -lc 'tail -n+10 README.md'"), vec![ParsedCommand::Read { - cmd: shlex_split_safe("tail -n+10 README.md"), + cmd: "tail -n+10 README.md".to_string(), name: "README.md".to_string(), }], ); @@ -853,7 +872,7 @@ mod tests { assert_parsed( &shlex_split_safe("pnpm test"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("pnpm test"), + cmd: "pnpm test".to_string(), }], ); } @@ -866,12 +885,10 @@ mod tests { &shlex_split_safe(inner), vec![ ParsedCommand::Unknown { - cmd: shlex_split_safe("cd codex-cli"), + cmd: "cd codex-cli".to_string(), }, ParsedCommand::Unknown { - cmd: shlex_split_safe( - "pnpm exec vitest run tests/file-tag-utils.test.ts --threads=false --passWithNoTests", - ), + cmd: "pnpm exec vitest run tests/file-tag-utils.test.ts '--threads=false' --passWithNoTests".to_string(), }, ], ); @@ -882,7 +899,7 @@ mod tests { assert_parsed( &shlex_split_safe("cargo test -p codex-core parse_command::"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("cargo test -p codex-core parse_command::"), + cmd: "cargo test -p codex-core parse_command::".to_string(), }], ); } @@ -894,9 +911,7 @@ mod tests { "cd core && cargo test -q parse_command::tests::bash_dash_c_pipeline_parsing parse_command::tests::fd_file_finder_variants", ), vec![ParsedCommand::Test { - cmd: shlex_split_safe( - "cargo test -q parse_command::tests::bash_dash_c_pipeline_parsing parse_command::tests::fd_file_finder_variants", - ), + cmd: "cargo test -q parse_command::tests::bash_dash_c_pipeline_parsing parse_command::tests::fd_file_finder_variants".to_string(), }], ); } @@ -906,7 +921,7 @@ mod tests { assert_parsed( &shlex_split_safe("cd core && cargo test -q parse_command::tests"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("cargo test -q parse_command::tests"), + cmd: "cargo test -q parse_command::tests".to_string(), }], ); } @@ -916,7 +931,7 @@ mod tests { assert_parsed( &shlex_split_safe("cd core && cargo test --all-features parse_command -- --nocapture"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("cargo test --all-features parse_command -- --nocapture"), + cmd: "cargo test --all-features parse_command -- --nocapture".to_string(), }], ); } @@ -928,7 +943,7 @@ mod tests { assert_parsed( &shlex_split_safe("black src"), vec![ParsedCommand::Format { - cmd: shlex_split_safe("black src"), + cmd: "black src".to_string(), tool: Some("black".to_string()), targets: Some(vec!["src".to_string()]), }], @@ -938,7 +953,7 @@ mod tests { assert_parsed( &shlex_split_safe("ruff check ."), vec![ParsedCommand::Lint { - cmd: shlex_split_safe("ruff check ."), + cmd: "ruff check .".to_string(), tool: Some("ruff".to_string()), targets: Some(vec![".".to_string()]), }], @@ -948,7 +963,7 @@ mod tests { assert_parsed( &shlex_split_safe("ruff format pkg/"), vec![ParsedCommand::Format { - cmd: shlex_split_safe("ruff format pkg/"), + cmd: "ruff format pkg/".to_string(), tool: Some("ruff".to_string()), targets: Some(vec!["pkg/".to_string()]), }], @@ -961,7 +976,7 @@ mod tests { assert_parsed( &shlex_split_safe("pnpm -r test"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("pnpm -r test"), + cmd: "pnpm -r test".to_string(), }], ); @@ -969,7 +984,7 @@ mod tests { assert_parsed( &shlex_split_safe("npm run format -- -w ."), vec![ParsedCommand::Format { - cmd: shlex_split_safe("npm run format -- -w ."), + cmd: "npm run format -- -w .".to_string(), tool: Some("npm-script:format".to_string()), targets: None, }], @@ -981,7 +996,7 @@ mod tests { assert_parsed( &shlex_split_safe("yarn test"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("yarn test"), + cmd: "yarn test".to_string(), }], ); } @@ -992,7 +1007,7 @@ mod tests { assert_parsed( &shlex_split_safe("pytest tests/test_example.py"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("pytest tests/test_example.py"), + cmd: "pytest tests/test_example.py".to_string(), }], ); @@ -1000,7 +1015,7 @@ mod tests { assert_parsed( &shlex_split_safe("go test ./... -run '^TestFoo$'"), vec![ParsedCommand::Test { - cmd: shlex_split_safe("go test ./... -run '^TestFoo$'"), + cmd: "go test ./... -run '^TestFoo$'".to_string(), }], ); } @@ -1010,7 +1025,7 @@ mod tests { assert_parsed( &shlex_split_safe("grep -R TODO src"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("grep -R TODO src"), + cmd: "grep -R TODO src".to_string(), query: Some("TODO".to_string()), path: Some("src".to_string()), }], @@ -1022,7 +1037,7 @@ mod tests { assert_parsed( &shlex_split_safe("rg --colors=never -n foo src"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --colors=never -n foo src"), + cmd: "rg '--colors=never' -n foo src".to_string(), query: Some("foo".to_string()), path: Some("src".to_string()), }], @@ -1035,7 +1050,7 @@ mod tests { assert_parsed( &shlex_split_safe("cat -- ./-strange-file-name"), vec![ParsedCommand::Read { - cmd: shlex_split_safe("cat -- ./-strange-file-name"), + cmd: "cat -- ./-strange-file-name".to_string(), name: "-strange-file-name".to_string(), }], ); @@ -1044,7 +1059,7 @@ mod tests { assert_parsed( &shlex_split_safe("sed -n '12,20p' Cargo.toml"), vec![ParsedCommand::Read { - cmd: shlex_split_safe("sed -n '12,20p' Cargo.toml"), + cmd: "sed -n '12,20p' Cargo.toml".to_string(), name: "Cargo.toml".to_string(), }], ); @@ -1056,7 +1071,7 @@ mod tests { assert_parsed( &shlex_split_safe("rg --files | nl -ba"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("rg --files"), + cmd: "rg --files".to_string(), query: None, path: None, }], @@ -1068,7 +1083,7 @@ mod tests { assert_parsed( &shlex_split_safe("ls --time-style=long-iso ./dist"), vec![ParsedCommand::ListFiles { - cmd: shlex_split_safe("ls --time-style=long-iso ./dist"), + cmd: "ls '--time-style=long-iso' ./dist".to_string(), // short_display_path drops "dist" and shows "." as the last useful segment path: Some(".".to_string()), }], @@ -1080,7 +1095,7 @@ mod tests { assert_parsed( &shlex_split_safe("eslint -c .eslintrc.json src"), vec![ParsedCommand::Lint { - cmd: shlex_split_safe("eslint -c .eslintrc.json src"), + cmd: "eslint -c .eslintrc.json src".to_string(), tool: Some("eslint".to_string()), targets: Some(vec!["src".to_string()]), }], @@ -1092,7 +1107,7 @@ mod tests { assert_parsed( &shlex_split_safe("npx eslint -c .eslintrc src"), vec![ParsedCommand::Lint { - cmd: shlex_split_safe("npx eslint -c .eslintrc src"), + cmd: "npx eslint -c .eslintrc src".to_string(), tool: Some("eslint".to_string()), targets: Some(vec!["src".to_string()]), }], @@ -1104,7 +1119,7 @@ mod tests { assert_parsed( &shlex_split_safe("fd -t f src/"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("fd -t f src/"), + cmd: "fd -t f src/".to_string(), query: None, path: Some("src".to_string()), }], @@ -1114,7 +1129,7 @@ mod tests { assert_parsed( &shlex_split_safe("fd main src"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("fd main src"), + cmd: "fd main src".to_string(), query: Some("main".to_string()), path: Some("src".to_string()), }], @@ -1126,7 +1141,7 @@ mod tests { assert_parsed( &shlex_split_safe("find . -name '*.rs'"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("find . -name '*.rs'"), + cmd: "find . -name '*.rs'".to_string(), query: Some("*.rs".to_string()), path: Some(".".to_string()), }], @@ -1138,7 +1153,7 @@ mod tests { assert_parsed( &shlex_split_safe("find src -type f"), vec![ParsedCommand::Search { - cmd: shlex_split_safe("find src -type f"), + cmd: "find src -type f".to_string(), query: None, path: Some("src".to_string()), }], @@ -1147,12 +1162,12 @@ mod tests { } pub fn parse_command_impl(command: &[String]) -> Vec { - let normalized = normalize_tokens(command); - - if let Some(commands) = parse_bash_lc_commands(command, &normalized) { + if let Some(commands) = parse_bash_lc_commands(command) { return commands; } + let normalized = normalize_tokens(command); + let parts = if contains_connectors(&normalized) { split_on_connectors(&normalized) } else { @@ -1163,55 +1178,77 @@ pub fn parse_command_impl(command: &[String]) -> Vec { // so summaries reflect the order they will run. // Map each pipeline segment to its parsed summary. - let mut parsed: Vec = parts + let mut commands: Vec = parts .iter() .map(|tokens| summarize_main_tokens(tokens)) .collect(); - // If a pipeline ends with `nl` using only flags (e.g., `| nl -ba`), drop it so the - // main action (e.g., a sed range over a file) is surfaced cleanly. - if parsed.len() >= 2 { - let has_and_and = normalized.iter().any(|t| t == "&&"); - let contains_test = parsed - .iter() - .any(|pc| matches!(pc, ParsedCommand::Test { .. })); - parsed.retain(|pc| match pc { - ParsedCommand::Unknown { cmd } => { - if let Some(first) = cmd.first() { - // Drop cosmetic echo segments in chained commands - if has_and_and && first == "echo" { - return false; - } - // In non-bash chained commands, ignore directory changes like `cd foo` - // when the sequence includes a recognized test command. Preserve `cd` - // for other sequences (e.g., followed by a search command). - if has_and_and && contains_test && first == "cd" { - return false; - } - // Drop no-op commands like `true` - if cmd.len() == 1 && first == "true" { - return false; - } - if first == "nl" { - // Treat `nl` without an explicit file operand as formatting-only. - return cmd.iter().skip(1).any(|a| !a.starts_with('-')); - } - } - true - } - _ => true, - }); + while let Some(next) = simplify_once(&commands) { + commands = next; + } + + commands +} + +fn simplify_once(commands: &[ParsedCommand]) -> Option> { + if commands.len() <= 1 { + return None; } - // Also drop standalone `true` commands when not part of a chained `&&` context above - parsed.retain(|pc| match pc { + // echo ... && ...rest => ...rest + if let ParsedCommand::Unknown { cmd } = &commands[0] + && shlex_split(cmd).is_some_and(|t| t.first().map(|s| s.as_str()) == Some("echo")) + { + return Some(commands[1..].to_vec()); + } + + // cd foo && [any Test command] => [any Test command] + if let Some(idx) = commands.iter().position(|pc| match pc { ParsedCommand::Unknown { cmd } => { - !(cmd.len() == 1 && cmd.first().is_some_and(|s| s == "true")) + shlex_split(cmd).is_some_and(|t| t.first().map(|s| s.as_str()) == Some("cd")) } - _ => true, - }); + _ => false, + }) && commands + .iter() + .skip(idx + 1) + .any(|pc| matches!(pc, ParsedCommand::Test { .. })) + { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } + + // cmd || true => cmd + if let Some(idx) = commands.iter().position(|pc| match pc { + ParsedCommand::Noop { cmd } => cmd == "true", + _ => false, + }) { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } + + // nl -[any_flags] && ...rest => ...rest + if let Some(idx) = commands.iter().position(|pc| match pc { + ParsedCommand::Unknown { cmd } => { + if let Some(tokens) = shlex_split(cmd) { + tokens.first().is_some_and(|s| s.as_str() == "nl") + && tokens.iter().skip(1).all(|t| t.starts_with('-')) + } else { + false + } + } + _ => false, + }) { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } - parsed + None } /// Validates that this is a `sed -n 123,123p` command. @@ -1497,19 +1534,19 @@ fn classify_npm_like(tool: &str, tail: &[String], full_cmd: &[String]) -> Option let lname = name.to_lowercase(); if lname == "test" || lname == "unit" || lname == "jest" || lname == "vitest" { return Some(ParsedCommand::Test { - cmd: full_cmd.to_vec(), + cmd: shlex_join(full_cmd), }); } if lname == "lint" || lname == "eslint" { return Some(ParsedCommand::Lint { - cmd: full_cmd.to_vec(), + cmd: shlex_join(full_cmd), tool: Some(format!("{tool}-script:{name}")), targets: None, }); } if lname == "format" || lname == "fmt" || lname == "prettier" { return Some(ParsedCommand::Format { - cmd: full_cmd.to_vec(), + cmd: shlex_join(full_cmd), tool: Some(format!("{tool}-script:{name}")), targets: None, }); @@ -1518,126 +1555,134 @@ fn classify_npm_like(tool: &str, tail: &[String], full_cmd: &[String]) -> Option None } -fn parse_bash_lc_commands( - original: &[String], - normalized: &[String], -) -> Option> { +fn parse_bash_lc_commands(original: &[String]) -> Option> { let [bash, flag, script] = original else { return None; }; if bash != "bash" || flag != "-lc" { return None; } - if let Some(tree) = try_parse_bash(script) { - if let Some(all_commands) = try_parse_word_only_commands_sequence(&tree, script) { - if !all_commands.is_empty() { - let script_tokens = shlex_split(script) - .unwrap_or_else(|| vec!["bash".to_string(), flag.clone(), script.clone()]); - // Strip small formatting helpers (e.g., head/tail/awk/wc/etc) so we - // bias toward the primary command when pipelines are present. - // First, drop obvious small formatting helpers (e.g., wc/awk/etc). - let had_multiple_commands = all_commands.len() > 1; - // The bash AST walker yields commands in right-to-left order for - // connector/pipeline sequences. Reverse to reflect actual execution order. - let mut filtered_commands = drop_small_formatting_commands(all_commands); - filtered_commands.reverse(); - if filtered_commands.is_empty() { - return Some(vec![ParsedCommand::Unknown { - cmd: normalized.to_vec(), - }]); - } - let mut commands: Vec = filtered_commands - .into_iter() - .map(|tokens| summarize_main_tokens(&tokens)) - .collect(); - // Drop no-op `true` commands - commands.retain(|pc| match pc { - ParsedCommand::Unknown { cmd } => { - !(cmd.len() == 1 && cmd.first().is_some_and(|s| s == "true")) - } - _ => true, - }); - commands = maybe_collapse_cat_sed(commands, &script_tokens); - if commands.len() == 1 { - // If we reduced to a single command, attribute the full original script - // for clearer UX in file-reading and listing scenarios, or when there were - // no connectors in the original script. For search commands that came from - // a pipeline (e.g. `rg --files | sed -n`), keep only the primary command. - let had_connectors = had_multiple_commands - || script_tokens - .iter() - .any(|t| t == "|" || t == "&&" || t == "||" || t == ";"); - commands = commands - .into_iter() - .map(|pc| match pc { - ParsedCommand::Read { name, cmd } => { - if had_connectors { - let has_pipe = script_tokens.iter().any(|t| t == "|"); - let has_sed_n = script_tokens.windows(2).any(|w| { - w.first().map(|s| s.as_str()) == Some("sed") - && w.get(1).map(|s| s.as_str()) == Some("-n") - }); - if has_pipe && has_sed_n { - ParsedCommand::Read { - cmd: script_tokens.clone(), - name, - } - } else { - ParsedCommand::Read { cmd, name } - } - } else { - ParsedCommand::Read { - cmd: script_tokens.clone(), - name, - } + if let Some(tree) = try_parse_bash(script) + && let Some(all_commands) = try_parse_word_only_commands_sequence(&tree, script) + && !all_commands.is_empty() + { + let script_tokens = shlex_split(script) + .unwrap_or_else(|| vec!["bash".to_string(), flag.clone(), script.clone()]); + // Strip small formatting helpers (e.g., head/tail/awk/wc/etc) so we + // bias toward the primary command when pipelines are present. + // First, drop obvious small formatting helpers (e.g., wc/awk/etc). + let had_multiple_commands = all_commands.len() > 1; + // The bash AST walker yields commands in right-to-left order for + // connector/pipeline sequences. Reverse to reflect actual execution order. + let mut filtered_commands = drop_small_formatting_commands(all_commands); + filtered_commands.reverse(); + if filtered_commands.is_empty() { + return Some(vec![ParsedCommand::Unknown { + cmd: script.clone(), + }]); + } + let mut commands: Vec = filtered_commands + .into_iter() + .map(|tokens| summarize_main_tokens(&tokens)) + .collect(); + if commands.len() > 1 { + commands.retain(|pc| !matches!(pc, ParsedCommand::Noop { .. })); + } + if commands.len() == 1 { + // If we reduced to a single command, attribute the full original script + // for clearer UX in file-reading and listing scenarios, or when there were + // no connectors in the original script. For search commands that came from + // a pipeline (e.g. `rg --files | sed -n`), keep only the primary command. + let had_connectors = had_multiple_commands + || script_tokens + .iter() + .any(|t| t == "|" || t == "&&" || t == "||" || t == ";"); + commands = commands + .into_iter() + .map(|pc| match pc { + ParsedCommand::Read { name, cmd, .. } => { + if had_connectors { + let has_pipe = script_tokens.iter().any(|t| t == "|"); + let has_sed_n = script_tokens.windows(2).any(|w| { + w.first().map(|s| s.as_str()) == Some("sed") + && w.get(1).map(|s| s.as_str()) == Some("-n") + }); + if has_pipe && has_sed_n { + ParsedCommand::Read { + cmd: script.clone(), + name, } - } - ParsedCommand::ListFiles { path, cmd } => { - if had_connectors { - ParsedCommand::ListFiles { cmd, path } - } else { - ParsedCommand::ListFiles { - cmd: script_tokens.clone(), - path, - } + } else { + ParsedCommand::Read { + cmd: cmd.clone(), + name, } } - ParsedCommand::Search { cmd, query, path } => { - if had_connectors { - ParsedCommand::Search { cmd, query, path } - } else { - ParsedCommand::Search { - cmd: script_tokens.clone(), - query, - path, - } - } + } else { + ParsedCommand::Read { + cmd: shlex_join(&script_tokens), + name, } - ParsedCommand::Format { tool, targets, .. } => ParsedCommand::Format { - cmd: script_tokens.clone(), - tool, - targets, - }, - ParsedCommand::Test { .. } => ParsedCommand::Test { - cmd: script_tokens.clone(), - }, - ParsedCommand::Lint { tool, targets, .. } => ParsedCommand::Lint { - cmd: script_tokens.clone(), - tool, - targets, - }, - ParsedCommand::Unknown { .. } => ParsedCommand::Unknown { - cmd: script_tokens.clone(), - }, - }) - .collect(); - } - return Some(commands); - } + } + } + ParsedCommand::ListFiles { path, cmd, .. } => { + if had_connectors { + ParsedCommand::ListFiles { + cmd: cmd.clone(), + path, + } + } else { + ParsedCommand::ListFiles { + cmd: shlex_join(&script_tokens), + path, + } + } + } + ParsedCommand::Search { + query, path, cmd, .. + } => { + if had_connectors { + ParsedCommand::Search { + cmd: cmd.clone(), + query, + path, + } + } else { + ParsedCommand::Search { + cmd: shlex_join(&script_tokens), + query, + path, + } + } + } + ParsedCommand::Format { + tool, targets, cmd, .. + } => ParsedCommand::Format { + cmd: cmd.clone(), + tool, + targets, + }, + ParsedCommand::Test { cmd, .. } => ParsedCommand::Test { cmd: cmd.clone() }, + ParsedCommand::Lint { + tool, targets, cmd, .. + } => ParsedCommand::Lint { + cmd: cmd.clone(), + tool, + targets, + }, + ParsedCommand::Unknown { .. } => ParsedCommand::Unknown { + cmd: script.clone(), + }, + ParsedCommand::Noop { .. } => ParsedCommand::Noop { + cmd: script.clone(), + }, + }) + .collect(); } + return Some(commands); } Some(vec![ParsedCommand::Unknown { - cmd: normalized.to_vec(), + cmd: script.clone(), }]) } @@ -1681,44 +1726,17 @@ fn drop_small_formatting_commands(mut commands: Vec>) -> Vec, - script_tokens: &[String], -) -> Vec { - if commands.len() < 2 { - return commands; - } - let drop_leading_sed = match (&commands[0], &commands[1]) { - (ParsedCommand::Unknown { cmd: sed_cmd }, ParsedCommand::Read { cmd: cat_cmd, .. }) => { - let is_sed_n = sed_cmd.first().map(|s| s.as_str()) == Some("sed") - && sed_cmd.get(1).map(|s| s.as_str()) == Some("-n") - && is_valid_sed_n_arg(sed_cmd.get(2).map(|s| s.as_str())) - && sed_cmd.len() == 3; - let is_cat_file = - cat_cmd.first().map(|s| s.as_str()) == Some("cat") && cat_cmd.len() == 2; - is_sed_n && is_cat_file - } - _ => false, - }; - if drop_leading_sed { - if let ParsedCommand::Read { name, .. } = &commands[1] { - return vec![ParsedCommand::Read { - cmd: script_tokens.to_vec(), - name: name.clone(), - }]; - } - } - commands -} - fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { match main_cmd.split_first() { + Some((head, tail)) if head == "true" && tail.is_empty() => ParsedCommand::Noop { + cmd: shlex_join(main_cmd), + }, // (sed-specific logic handled below in dedicated arm returning Read) Some((head, tail)) if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("fmt") => { ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("cargo fmt".to_string()), targets: collect_non_flag_targets(&tail[1..]), } @@ -1727,7 +1745,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("clippy") => { ParsedCommand::Lint { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("cargo clippy".to_string()), targets: collect_non_flag_targets(&tail[1..]), } @@ -1736,45 +1754,45 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("test") => { ParsedCommand::Test { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } Some((head, tail)) if head == "rustfmt" => ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("rustfmt".to_string()), targets: collect_non_flag_targets(tail), }, Some((head, tail)) if head == "go" && tail.first().map(|s| s.as_str()) == Some("fmt") => { ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("go fmt".to_string()), targets: collect_non_flag_targets(&tail[1..]), } } Some((head, tail)) if head == "go" && tail.first().map(|s| s.as_str()) == Some("test") => { ParsedCommand::Test { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } Some((head, _)) if head == "pytest" => ParsedCommand::Test { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), }, Some((head, tail)) if head == "eslint" => { // Treat configuration flags with values (e.g. `-c .eslintrc`) as non-targets. let targets = collect_non_flag_targets_with_flags(tail, ESLINT_FLAGS_WITH_VALUES); ParsedCommand::Lint { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("eslint".to_string()), targets, } } Some((head, tail)) if head == "prettier" => ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("prettier".to_string()), targets: collect_non_flag_targets(tail), }, Some((head, tail)) if head == "black" => ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("black".to_string()), targets: collect_non_flag_targets(tail), }, @@ -1782,7 +1800,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if head == "ruff" && tail.first().map(|s| s.as_str()) == Some("check") => { ParsedCommand::Lint { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("ruff".to_string()), targets: collect_non_flag_targets(&tail[1..]), } @@ -1791,20 +1809,20 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if head == "ruff" && tail.first().map(|s| s.as_str()) == Some("format") => { ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("ruff".to_string()), targets: collect_non_flag_targets(&tail[1..]), } } Some((head, _)) if (head == "jest" || head == "vitest") => ParsedCommand::Test { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), }, Some((head, tail)) if head == "npx" && tail.first().map(|s| s.as_str()) == Some("eslint") => { let targets = collect_non_flag_targets_with_flags(&tail[1..], ESLINT_FLAGS_WITH_VALUES); ParsedCommand::Lint { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("eslint".to_string()), targets, } @@ -1813,7 +1831,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if head == "npx" && tail.first().map(|s| s.as_str()) == Some("prettier") => { ParsedCommand::Format { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), tool: Some("prettier".to_string()), targets: collect_non_flag_targets(&tail[1..]), } @@ -1824,7 +1842,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { cmd } else { ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } } @@ -1847,7 +1865,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { .find(|p| !p.starts_with('-')) .map(|p| short_display_path(p)); ParsedCommand::ListFiles { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), path, } } @@ -1867,7 +1885,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { ) }; ParsedCommand::Search { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), query, path, } @@ -1875,7 +1893,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { Some((head, tail)) if head == "fd" => { let (query, path) = parse_fd_query_and_path(tail); ParsedCommand::Search { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), query, path, } @@ -1884,7 +1902,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { // Basic find support: capture path and common name filter let (query, path) = parse_find_query_and_path(tail); ParsedCommand::Search { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), query, path, } @@ -1900,7 +1918,7 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { let query = non_flags.first().cloned().map(|s| s.to_string()); let path = non_flags.get(1).map(|s| short_display_path(s)); ParsedCommand::Search { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), query, path, } @@ -1915,12 +1933,12 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if effective_tail.len() == 1 { let name = short_display_path(&effective_tail[0]); ParsedCommand::Read { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), name, } } else { ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } } @@ -1953,13 +1971,13 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { let name = short_display_path(p); return ParsedCommand::Read { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), name, }; } } ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } Some((head, tail)) if head == "tail" => { @@ -1995,13 +2013,13 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { let name = short_display_path(p); return ParsedCommand::Read { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), name, }; } } ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } Some((head, tail)) if head == "nl" => { @@ -2010,12 +2028,12 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { let name = short_display_path(p); ParsedCommand::Read { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), name, } } else { ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } } @@ -2028,18 +2046,18 @@ fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { if let Some(path) = tail.get(2) { let name = short_display_path(path); ParsedCommand::Read { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), name, } } else { ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), } } } // Other commands _ => ParsedCommand::Unknown { - cmd: main_cmd.to_vec(), + cmd: shlex_join(main_cmd), }, } } diff --git a/codex-rs/core/src/plan_tool.rs b/codex-rs/core/src/plan_tool.rs index 9e81abcd882..bc39e4f6ce7 100644 --- a/codex-rs/core/src/plan_tool.rs +++ b/codex-rs/core/src/plan_tool.rs @@ -1,9 +1,6 @@ use std::collections::BTreeMap; use std::sync::LazyLock; -use serde::Deserialize; -use serde::Serialize; - use crate::codex::Session; use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; @@ -13,29 +10,13 @@ use crate::openai_tools::ResponsesApiTool; use crate::protocol::Event; use crate::protocol::EventMsg; -// Types for the TODO tool arguments matching codex-vscode/todo-mcp/src/main.rs -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StepStatus { - Pending, - InProgress, - Completed, -} +// Use the canonical plan tool types from the protocol crate to ensure +// type-identity matches events transported via `codex_protocol`. +pub use codex_protocol::plan_tool::PlanItemArg; +pub use codex_protocol::plan_tool::StepStatus; +pub use codex_protocol::plan_tool::UpdatePlanArgs; -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PlanItemArg { - pub step: String, - pub status: StepStatus, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct UpdatePlanArgs { - #[serde(default)] - pub explanation: Option, - pub plan: Vec, -} +// Types for the TODO tool arguments matching codex-vscode/todo-mcp/src/main.rs pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { let mut plan_item_props = BTreeMap::new(); diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index 9f46159d1d9..a869b40cb36 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -1,18 +1,19 @@ //! Project-level documentation discovery. //! -//! Project-level documentation can be stored in a file named `AGENTS.md`. -//! Currently, we include only the contents of the first file found as follows: +//! Project-level documentation can be stored in files named `AGENTS.md`. +//! We include the concatenation of all files found along the path from the +//! repository root to the current working directory as follows: //! -//! 1. Look for the doc file in the current working directory (as determined -//! by the `Config`). -//! 2. If not found, walk *upwards* until the Git repository root is reached -//! (detected by the presence of a `.git` directory/file), or failing that, -//! the filesystem root. -//! 3. If the Git root is encountered, look for the doc file there. If it -//! exists, the search stops – we do **not** walk past the Git root. +//! 1. Determine the Git repository root by walking upwards from the current +//! working directory until a `.git` directory or file is found. If no Git +//! root is found, only the current working directory is considered. +//! 2. Collect every `AGENTS.md` found from the repository root down to the +//! current working directory (inclusive) and concatenate their contents in +//! that order. +//! 3. We do **not** walk past the Git root. use crate::config::Config; -use std::path::Path; +use std::path::PathBuf; use tokio::io::AsyncReadExt; use tracing::error; @@ -26,7 +27,7 @@ const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; /// Combines `Config::instructions` and `AGENTS.md` (if present) into a single /// string of instructions. pub(crate) async fn get_user_instructions(config: &Config) -> Option { - match find_project_doc(config).await { + match read_project_docs(config).await { Ok(Some(project_doc)) => match &config.user_instructions { Some(original_instructions) => Some(format!( "{original_instructions}{PROJECT_DOC_SEPARATOR}{project_doc}" @@ -41,101 +42,139 @@ pub(crate) async fn get_user_instructions(config: &Config) -> Option { } } -/// Attempt to locate and load the project documentation. Currently, the search -/// starts from `Config::cwd`, but if we may want to consider other directories -/// in the future, e.g., additional writable directories in the `SandboxPolicy`. +/// Attempt to locate and load the project documentation. /// -/// On success returns `Ok(Some(contents))`. If no documentation file is found -/// the function returns `Ok(None)`. Unexpected I/O failures bubble up as -/// `Err` so callers can decide how to handle them. -async fn find_project_doc(config: &Config) -> std::io::Result> { - let max_bytes = config.project_doc_max_bytes; - - // Attempt to load from the working directory first. - if let Some(doc) = load_first_candidate(&config.cwd, CANDIDATE_FILENAMES, max_bytes).await? { - return Ok(Some(doc)); +/// On success returns `Ok(Some(contents))` where `contents` is the +/// concatenation of all discovered docs. If no documentation file is found the +/// function returns `Ok(None)`. Unexpected I/O failures bubble up as `Err` so +/// callers can decide how to handle them. +pub async fn read_project_docs(config: &Config) -> std::io::Result> { + let max_total = config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); } - // Walk up towards the filesystem root, stopping once we encounter the Git - // repository root. The presence of **either** a `.git` *file* or - // *directory* counts. - let mut dir = config.cwd.clone(); - - // Canonicalize the path so that we do not end up in an infinite loop when - // `cwd` contains `..` components. - if let Ok(canon) = dir.canonicalize() { - dir = canon; + let paths = discover_project_doc_paths(config)?; + if paths.is_empty() { + return Ok(None); } - while let Some(parent) = dir.parent() { - // `.git` can be a *file* (for worktrees or submodules) or a *dir*. - let git_marker = dir.join(".git"); - let git_exists = match tokio::fs::metadata(&git_marker).await { - Ok(_) => true, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, - Err(e) => return Err(e), - }; + let mut remaining: u64 = max_total as u64; + let mut parts: Vec = Vec::new(); - if git_exists { - // We are at the repo root – attempt one final load. - if let Some(doc) = load_first_candidate(&dir, CANDIDATE_FILENAMES, max_bytes).await? { - return Ok(Some(doc)); - } + for p in paths { + if remaining == 0 { break; } - dir = parent.to_path_buf(); - } - - Ok(None) -} - -/// Attempt to load the first candidate file found in `dir`. Returns the file -/// contents (truncated if it exceeds `max_bytes`) when successful. -async fn load_first_candidate( - dir: &Path, - names: &[&str], - max_bytes: usize, -) -> std::io::Result> { - for name in names { - let candidate = dir.join(name); - - let file = match tokio::fs::File::open(&candidate).await { + let file = match tokio::fs::File::open(&p).await { + Ok(f) => f, Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, Err(e) => return Err(e), - Ok(f) => f, }; let size = file.metadata().await?.len(); + let mut reader = tokio::io::BufReader::new(file).take(remaining); + let mut data: Vec = Vec::new(); + reader.read_to_end(&mut data).await?; - let reader = tokio::io::BufReader::new(file); - let mut data = Vec::with_capacity(std::cmp::min(size as usize, max_bytes)); - let mut limited = reader.take(max_bytes as u64); - limited.read_to_end(&mut data).await?; - - if size as usize > max_bytes { + if size > remaining { tracing::warn!( - "Project doc `{}` exceeds {max_bytes} bytes - truncating.", - candidate.display(), + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, ); } - let contents = String::from_utf8_lossy(&data).to_string(); - if contents.trim().is_empty() { - // Empty file – treat as not found. - continue; + let text = String::from_utf8_lossy(&data).to_string(); + if !text.trim().is_empty() { + parts.push(text); + remaining = remaining.saturating_sub(data.len() as u64); } + } + + if parts.is_empty() { + Ok(None) + } else { + Ok(Some(parts.join("\n\n"))) + } +} - return Ok(Some(contents)); +/// Discover the list of AGENTS.md files using the same search rules as +/// `read_project_docs`, but return the file paths instead of concatenated +/// contents. The list is ordered from repository root to the current working +/// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` +/// is zero, returns an empty list. +pub fn discover_project_doc_paths(config: &Config) -> std::io::Result> { + let mut dir = config.cwd.clone(); + if let Ok(canon) = dir.canonicalize() { + dir = canon; } - Ok(None) + // Build chain from cwd upwards and detect git root. + let mut chain: Vec = vec![dir.clone()]; + let mut git_root: Option = None; + let mut cursor = dir.clone(); + while let Some(parent) = cursor.parent() { + let git_marker = cursor.join(".git"); + let git_exists = match std::fs::metadata(&git_marker) { + Ok(_) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(e) => return Err(e), + }; + + if git_exists { + git_root = Some(cursor.clone()); + break; + } + + chain.push(parent.to_path_buf()); + cursor = parent.to_path_buf(); + } + + let search_dirs: Vec = if let Some(root) = git_root { + let mut dirs: Vec = Vec::new(); + let mut saw_root = false; + for p in chain.iter().rev() { + if !saw_root { + if p == &root { + saw_root = true; + } else { + continue; + } + } + dirs.push(p.clone()); + } + dirs + } else { + vec![config.cwd.clone()] + }; + + let mut found: Vec = Vec::new(); + for d in search_dirs { + for name in CANDIDATE_FILENAMES { + let candidate = d.join(name); + match std::fs::symlink_metadata(&candidate) { + Ok(md) => { + let ft = md.file_type(); + // Allow regular files and symlinks; opening will later fail for dangling links. + if ft.is_file() || ft.is_symlink() { + found.push(candidate); + break; + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + } + } + } + + Ok(found) } #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] - use super::*; use crate::config::ConfigOverrides; use crate::config::ConfigToml; @@ -280,4 +319,32 @@ mod tests { assert_eq!(res, Some(INSTRUCTIONS.to_string())); } + + /// When both the repository root and the working directory contain + /// AGENTS.md files, their contents are concatenated from root to cwd. + #[tokio::test] + async fn concatenates_root_and_cwd_docs() { + let repo = tempfile::tempdir().expect("tempdir"); + + // Simulate a git repository. + std::fs::write( + repo.path().join(".git"), + "gitdir: /path/to/actual/git/dir\n", + ) + .unwrap(); + + // Repo root doc. + fs::write(repo.path().join("AGENTS.md"), "root doc").unwrap(); + + // Nested working directory with its own doc. + let nested = repo.path().join("workspace/crate_a"); + std::fs::create_dir_all(&nested).unwrap(); + fs::write(nested.join("AGENTS.md"), "crate doc").unwrap(); + + let mut cfg = make_config(&repo, 4096, None); + cfg.cwd = nested; + + let res = get_user_instructions(&cfg).await.expect("doc expected"); + assert_eq!(res, "root doc\n\ncrate doc"); + } } diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index 860a728def8..37cb8f4e4ce 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -21,7 +21,7 @@ pub enum SafetyCheck { pub fn assess_patch_safety( action: &ApplyPatchAction, policy: AskForApproval, - writable_roots: &[PathBuf], + sandbox_policy: &SandboxPolicy, cwd: &Path, ) -> SafetyCheck { if action.is_empty() { @@ -45,7 +45,7 @@ pub fn assess_patch_safety( // is possible that paths in the patch are hard links to files outside the // writable roots, so we should still run `apply_patch` in a sandbox in that // case. - if is_write_patch_constrained_to_writable_paths(action, writable_roots, cwd) + if is_write_patch_constrained_to_writable_paths(action, sandbox_policy, cwd) || policy == AskForApproval::OnFailure { // Only auto‑approve when we can actually enforce a sandbox. Otherwise @@ -171,13 +171,19 @@ pub fn get_platform_sandbox() -> Option { fn is_write_patch_constrained_to_writable_paths( action: &ApplyPatchAction, - writable_roots: &[PathBuf], + sandbox_policy: &SandboxPolicy, cwd: &Path, ) -> bool { // Early‑exit if there are no declared writable roots. - if writable_roots.is_empty() { - return false; - } + let writable_roots = match sandbox_policy { + SandboxPolicy::ReadOnly => { + return false; + } + SandboxPolicy::DangerFullAccess => { + return true; + } + SandboxPolicy::WorkspaceWrite { .. } => sandbox_policy.get_writable_roots_with_cwd(cwd), + }; // Normalize a path by removing `.` and resolving `..` without touching the // filesystem (works even if the file does not exist). @@ -209,15 +215,9 @@ fn is_write_patch_constrained_to_writable_paths( None => return false, }; - writable_roots.iter().any(|root| { - let root_abs = if root.is_absolute() { - root.clone() - } else { - normalize(&cwd.join(root)).unwrap_or_else(|| cwd.join(root)) - }; - - abs.starts_with(&root_abs) - }) + writable_roots + .iter() + .any(|writable_root| writable_root.is_path_writable(&abs)) }; for (path, change) in action.changes() { @@ -231,10 +231,10 @@ fn is_write_patch_constrained_to_writable_paths( if !is_path_writable(path) { return false; } - if let Some(dest) = move_path { - if !is_path_writable(dest) { - return false; - } + if let Some(dest) = move_path + && !is_path_writable(dest) + { + return false; } } } @@ -245,40 +245,57 @@ fn is_write_patch_constrained_to_writable_paths( #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; + use tempfile::TempDir; #[test] fn test_writable_roots_constraint() { - let cwd = std::env::current_dir().unwrap(); + // Use a temporary directory as our workspace to avoid touching + // the real current working directory. + let tmp = TempDir::new().unwrap(); + let cwd = tmp.path().to_path_buf(); let parent = cwd.parent().unwrap().to_path_buf(); - // Helper to build a single‑entry map representing a patch that adds a - // file at `p`. + // Helper to build a single‑entry patch that adds a file at `p`. let make_add_change = |p: PathBuf| ApplyPatchAction::new_add_for_test(&p, "".to_string()); let add_inside = make_add_change(cwd.join("inner.txt")); let add_outside = make_add_change(parent.join("outside.txt")); + // Policy limited to the workspace only; exclude system temp roots so + // only `cwd` is writable by default. + let policy_workspace_only = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + assert!(is_write_patch_constrained_to_writable_paths( &add_inside, - &[PathBuf::from(".")], + &policy_workspace_only, &cwd, )); - let add_outside_2 = make_add_change(parent.join("outside.txt")); assert!(!is_write_patch_constrained_to_writable_paths( - &add_outside_2, - &[PathBuf::from(".")], + &add_outside, + &policy_workspace_only, &cwd, )); - // With parent dir added as writable root, it should pass. + // With the parent dir explicitly added as a writable root, the + // outside write should be permitted. + let policy_with_parent = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![parent.clone()], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; assert!(is_write_patch_constrained_to_writable_paths( &add_outside, - &[PathBuf::from("..")], + &policy_with_parent, &cwd, - )) + )); } #[test] diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs index ff9dbaa7f9a..31aa770d2ec 100644 --- a/codex-rs/core/src/seatbelt.rs +++ b/codex-rs/core/src/seatbelt.rs @@ -122,7 +122,6 @@ fn create_seatbelt_command_args( #[cfg(test)] mod tests { - #![expect(clippy::expect_used)] use super::MACOS_SEATBELT_BASE_POLICY; use super::create_seatbelt_command_args; use crate::protocol::SandboxPolicy; diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index cc58eb74606..3a69874eb2b 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -1,14 +1,24 @@ +use serde::Deserialize; +use serde::Serialize; use shlex; +use std::path::PathBuf; -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct ZshShell { shell_path: String, zshrc_path: String, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct PowerShellConfig { + exe: String, // Executable name or path, e.g. "pwsh" or "powershell.exe". + bash_exe_fallback: Option, // In case the model generates a bash command. +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub enum Shell { Zsh(ZshShell), + PowerShell(PowerShellConfig), Unknown, } @@ -33,6 +43,61 @@ impl Shell { } Some(result) } + Shell::PowerShell(ps) => { + // If model generated a bash command, prefer a detected bash fallback + if let Some(script) = strip_bash_lc(&command) { + return match &ps.bash_exe_fallback { + Some(bash) => Some(vec![ + bash.to_string_lossy().to_string(), + "-lc".to_string(), + script, + ]), + + // No bash fallback → run the script under PowerShell. + // It will likely fail (except for some simple commands), but the error + // should give a clue to the model to fix upon retry that it's running under PowerShell. + None => Some(vec![ + ps.exe.clone(), + "-NoProfile".to_string(), + "-Command".to_string(), + script, + ]), + }; + } + + // Not a bash command. If model did not generate a PowerShell command, + // turn it into a PowerShell command. + let first = command.first().map(String::as_str); + if first != Some(ps.exe.as_str()) { + // TODO (CODEX_2900): Handle escaping newlines. + if command.iter().any(|a| a.contains('\n') || a.contains('\r')) { + return Some(command); + } + + let joined = shlex::try_join(command.iter().map(|s| s.as_str())).ok(); + return joined.map(|arg| { + vec![ + ps.exe.clone(), + "-NoProfile".to_string(), + "-Command".to_string(), + arg, + ] + }); + } + + // Model generated a PowerShell command. Run it. + Some(command) + } + Shell::Unknown => None, + } + } + + pub fn name(&self) -> Option { + match self { + Shell::Zsh(zsh) => std::path::Path::new(&zsh.shell_path) + .file_name() + .map(|s| s.to_string_lossy().to_string()), + Shell::PowerShell(ps) => Some(ps.exe.clone()), Shell::Unknown => None, } } @@ -70,13 +135,13 @@ pub async fn default_user_shell() -> Shell { } let stdout = String::from_utf8_lossy(&o.stdout); for line in stdout.lines() { - if let Some(shell_path) = line.strip_prefix("UserShell: ") { - if shell_path.ends_with("/zsh") { - return Shell::Zsh(ZshShell { - shell_path: shell_path.to_string(), - zshrc_path: format!("{home}/.zshrc"), - }); - } + if let Some(shell_path) = line.strip_prefix("UserShell: ") + && shell_path.ends_with("/zsh") + { + return Shell::Zsh(ZshShell { + shell_path: shell_path.to_string(), + zshrc_path: format!("{home}/.zshrc"), + }); } } @@ -86,11 +151,51 @@ pub async fn default_user_shell() -> Shell { } } -#[cfg(not(target_os = "macos"))] +#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] pub async fn default_user_shell() -> Shell { Shell::Unknown } +#[cfg(target_os = "windows")] +pub async fn default_user_shell() -> Shell { + use tokio::process::Command; + + // Prefer PowerShell 7+ (`pwsh`) if available, otherwise fall back to Windows PowerShell. + let has_pwsh = Command::new("pwsh") + .arg("-NoLogo") + .arg("-NoProfile") + .arg("-Command") + .arg("$PSVersionTable.PSVersion.Major") + .output() + .await + .map(|o| o.status.success()) + .unwrap_or(false); + let bash_exe = if Command::new("bash.exe") + .arg("--version") + .output() + .await + .ok() + .map(|o| o.status.success()) + .unwrap_or(false) + { + which::which("bash.exe").ok() + } else { + None + }; + + if has_pwsh { + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: bash_exe, + }) + } else { + Shell::PowerShell(PowerShellConfig { + exe: "powershell.exe".to_string(), + bash_exe_fallback: bash_exe, + }) + } +} + #[cfg(test)] #[cfg(target_os = "macos")] mod tests { @@ -98,7 +203,6 @@ mod tests { use std::process::Command; #[tokio::test] - #[expect(clippy::unwrap_used)] async fn test_current_shell_detects_zsh() { let shell = Command::new("sh") .arg("-c") @@ -129,7 +233,6 @@ mod tests { assert_eq!(actual_cmd, None); } - #[expect(clippy::unwrap_used)] #[tokio::test] async fn test_run_with_profile_escaping_and_execution() { let shell_path = "/bin/zsh"; @@ -167,9 +270,6 @@ mod tests { for (input, expected_cmd, expected_output) in cases { use std::collections::HashMap; use std::path::PathBuf; - use std::sync::Arc; - - use tokio::sync::Notify; use crate::exec::ExecParams; use crate::exec::SandboxType; @@ -219,7 +319,6 @@ mod tests { justification: None, }, SandboxType::None, - Arc::new(Notify::new()), &SandboxPolicy::DangerFullAccess, &None, None, @@ -237,3 +336,97 @@ mod tests { } } } + +#[cfg(test)] +#[cfg(target_os = "windows")] +mod tests_windows { + use super::*; + + #[test] + fn test_format_default_shell_invocation_powershell() { + let cases = vec![ + ( + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: None, + }), + vec!["bash", "-lc", "echo hello"], + vec!["pwsh.exe", "-NoProfile", "-Command", "echo hello"], + ), + ( + Shell::PowerShell(PowerShellConfig { + exe: "powershell.exe".to_string(), + bash_exe_fallback: None, + }), + vec!["bash", "-lc", "echo hello"], + vec!["powershell.exe", "-NoProfile", "-Command", "echo hello"], + ), + ( + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: Some(PathBuf::from("bash.exe")), + }), + vec!["bash", "-lc", "echo hello"], + vec!["bash.exe", "-lc", "echo hello"], + ), + ( + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: Some(PathBuf::from("bash.exe")), + }), + vec![ + "bash", + "-lc", + "apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: destination_file.txt\n-original content\n+modified content\n*** End Patch\nEOF", + ], + vec![ + "bash.exe", + "-lc", + "apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: destination_file.txt\n-original content\n+modified content\n*** End Patch\nEOF", + ], + ), + ( + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: Some(PathBuf::from("bash.exe")), + }), + vec!["echo", "hello"], + vec!["pwsh.exe", "-NoProfile", "-Command", "echo hello"], + ), + ( + Shell::PowerShell(PowerShellConfig { + exe: "pwsh.exe".to_string(), + bash_exe_fallback: Some(PathBuf::from("bash.exe")), + }), + vec!["pwsh.exe", "-NoProfile", "-Command", "echo hello"], + vec!["pwsh.exe", "-NoProfile", "-Command", "echo hello"], + ), + ( + // TODO (CODEX_2900): Handle escaping newlines for powershell invocation. + Shell::PowerShell(PowerShellConfig { + exe: "powershell.exe".to_string(), + bash_exe_fallback: Some(PathBuf::from("bash.exe")), + }), + vec![ + "codex-mcp-server.exe", + "--codex-run-as-apply-patch", + "*** Begin Patch\n*** Update File: C:\\Users\\person\\destination_file.txt\n-original content\n+modified content\n*** End Patch", + ], + vec![ + "codex-mcp-server.exe", + "--codex-run-as-apply-patch", + "*** Begin Patch\n*** Update File: C:\\Users\\person\\destination_file.txt\n-original content\n+modified content\n*** End Patch", + ], + ), + ]; + + for (shell, input, expected_cmd) in cases { + let actual_cmd = shell + .format_default_shell_invocation(input.iter().map(|s| s.to_string()).collect()); + assert_eq!( + actual_cmd, + Some(expected_cmd.iter().map(|s| s.to_string()).collect()) + ); + } + } +} diff --git a/codex-rs/core/src/terminal.rs b/codex-rs/core/src/terminal.rs new file mode 100644 index 00000000000..02104f8be5c --- /dev/null +++ b/codex-rs/core/src/terminal.rs @@ -0,0 +1,72 @@ +use std::sync::OnceLock; + +static TERMINAL: OnceLock = OnceLock::new(); + +pub fn user_agent() -> String { + TERMINAL.get_or_init(detect_terminal).to_string() +} + +/// Sanitize a header value to be used in a User-Agent string. +/// +/// This function replaces any characters that are not allowed in a User-Agent string with an underscore. +/// +/// # Arguments +/// +/// * `value` - The value to sanitize. +fn is_valid_header_value_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '/' +} + +fn sanitize_header_value(value: String) -> String { + value.replace(|c| !is_valid_header_value_char(c), "_") +} + +fn detect_terminal() -> String { + sanitize_header_value( + if let Ok(tp) = std::env::var("TERM_PROGRAM") + && !tp.trim().is_empty() + { + let ver = std::env::var("TERM_PROGRAM_VERSION").ok(); + match ver { + Some(v) if !v.trim().is_empty() => format!("{tp}/{v}"), + _ => tp, + } + } else if let Ok(v) = std::env::var("WEZTERM_VERSION") { + if !v.trim().is_empty() { + format!("WezTerm/{v}") + } else { + "WezTerm".to_string() + } + } else if std::env::var("KITTY_WINDOW_ID").is_ok() + || std::env::var("TERM") + .map(|t| t.contains("kitty")) + .unwrap_or(false) + { + "kitty".to_string() + } else if std::env::var("ALACRITTY_SOCKET").is_ok() + || std::env::var("TERM") + .map(|t| t == "alacritty") + .unwrap_or(false) + { + "Alacritty".to_string() + } else if let Ok(v) = std::env::var("KONSOLE_VERSION") { + if !v.trim().is_empty() { + format!("Konsole/{v}") + } else { + "Konsole".to_string() + } + } else if std::env::var("GNOME_TERMINAL_SCREEN").is_ok() { + return "gnome-terminal".to_string(); + } else if let Ok(v) = std::env::var("VTE_VERSION") { + if !v.trim().is_empty() { + format!("VTE/{v}") + } else { + "VTE".to_string() + } + } else if std::env::var("WT_SESSION").is_ok() { + return "WindowsTerminal".to_string(); + } else { + std::env::var("TERM").unwrap_or_else(|_| "unknown".to_string()) + }, + ) +} diff --git a/codex-rs/core/src/turn_diff_tracker.rs b/codex-rs/core/src/turn_diff_tracker.rs index 7026d7bb325..5e73fa0650b 100644 --- a/codex-rs/core/src/turn_diff_tracker.rs +++ b/codex-rs/core/src/turn_diff_tracker.rs @@ -466,7 +466,6 @@ fn is_windows_drive_or_unc_root(p: &std::path::Path) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; use pretty_assertions::assert_eq; use tempfile::tempdir; diff --git a/codex-rs/core/src/user_agent.rs b/codex-rs/core/src/user_agent.rs index a0cf3870693..a63170cebdc 100644 --- a/codex-rs/core/src/user_agent.rs +++ b/codex-rs/core/src/user_agent.rs @@ -4,16 +4,16 @@ pub fn get_codex_user_agent(originator: Option<&str>) -> String { let build_version = env!("CARGO_PKG_VERSION"); let os_info = os_info::get(); format!( - "{}/{build_version} ({} {}; {})", + "{}/{build_version} ({} {}; {}) {}", originator.unwrap_or(DEFAULT_ORIGINATOR), os_info.os_type(), os_info.version(), os_info.architecture().unwrap_or("unknown"), + crate::terminal::user_agent() ) } #[cfg(test)] -#[allow(clippy::unwrap_used)] mod tests { use super::*; @@ -28,9 +28,10 @@ mod tests { fn test_macos() { use regex_lite::Regex; let user_agent = get_codex_user_agent(None); - let re = - Regex::new(r"^codex_cli_rs/\d+\.\d+\.\d+ \(Mac OS \d+\.\d+\.\d+; (x86_64|arm64)\)$") - .unwrap(); + let re = Regex::new( + r"^codex_cli_rs/\d+\.\d+\.\d+ \(Mac OS \d+\.\d+\.\d+; (x86_64|arm64)\) (\S+)$", + ) + .unwrap(); assert!(re.is_match(&user_agent)); } } diff --git a/codex-rs/core/src/user_notification.rs b/codex-rs/core/src/user_notification.rs index e7479f89cdb..0a3cb49e782 100644 --- a/codex-rs/core/src/user_notification.rs +++ b/codex-rs/core/src/user_notification.rs @@ -20,7 +20,6 @@ pub(crate) enum UserNotification { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; #[test] diff --git a/codex-rs/core/src/util.rs b/codex-rs/core/src/util.rs index fb5f45de6fc..e1248a4867d 100644 --- a/codex-rs/core/src/util.rs +++ b/codex-rs/core/src/util.rs @@ -1,32 +1,11 @@ use std::path::Path; -use std::sync::Arc; use std::time::Duration; use rand::Rng; -use tokio::sync::Notify; -use tracing::debug; const INITIAL_DELAY_MS: u64 = 200; const BACKOFF_FACTOR: f64 = 2.0; -/// Make a CancellationToken that is fulfilled when SIGINT occurs. -pub fn notify_on_sigint() -> Arc { - let notify = Arc::new(Notify::new()); - - tokio::spawn({ - let notify = Arc::clone(¬ify); - async move { - loop { - tokio::signal::ctrl_c().await.ok(); - debug!("Keyboard interrupt"); - notify.notify_waiters(); - } - } - }); - - notify -} - pub(crate) fn backoff(attempt: u64) -> Duration { let exp = BACKOFF_FACTOR.powi(attempt.saturating_sub(1) as i32); let base = (INITIAL_DELAY_MS as f64 * exp) as u64; diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs index c8570577ce0..dd53a8d3027 100644 --- a/codex-rs/core/tests/cli_stream.rs +++ b/codex-rs/core/tests/cli_stream.rs @@ -1,5 +1,3 @@ -#![expect(clippy::unwrap_used)] - use assert_cmd::Command as AssertCommand; use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use std::time::Duration; @@ -299,13 +297,12 @@ async fn integration_creates_and_checks_session_file() { Ok(v) => v, Err(_) => continue, }; - if item.get("type").and_then(|t| t.as_str()) == Some("message") { - if let Some(c) = item.get("content") { - if c.to_string().contains(&marker) { - matching_path = Some(path.to_path_buf()); - break; - } - } + if item.get("type").and_then(|t| t.as_str()) == Some("message") + && let Some(c) = item.get("content") + && c.to_string().contains(&marker) + { + matching_path = Some(path.to_path_buf()); + break; } } } @@ -378,13 +375,12 @@ async fn integration_creates_and_checks_session_file() { let Ok(item) = serde_json::from_str::(line) else { continue; }; - if item.get("type").and_then(|t| t.as_str()) == Some("message") { - if let Some(c) = item.get("content") { - if c.to_string().contains(&marker) { - found_message = true; - break; - } - } + if item.get("type").and_then(|t| t.as_str()) == Some("message") + && let Some(c) = item.get("content") + && c.to_string().contains(&marker) + { + found_message = true; + break; } } assert!( diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index bd7ad2feef9..30ba62eeaef 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -1,15 +1,13 @@ -#![allow(clippy::expect_used, clippy::unwrap_used)] - -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; +use codex_core::NewConversation; use codex_core::WireApi; use codex_core::built_in_model_providers; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; -use codex_core::protocol::SessionConfiguredEvent; use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use codex_login::AuthMode; use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id; @@ -28,10 +26,12 @@ fn sse_completed(id: &str) -> String { load_sse_fixture_with_id("tests/fixtures/completed_template.json", id) } +#[expect(clippy::unwrap_used)] fn assert_message_role(request_body: &serde_json::Value, role: &str) { assert_eq!(request_body["role"].as_str().unwrap(), role); } +#[expect(clippy::expect_used)] fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { let content = request_body["content"][0]["text"] .as_str() @@ -43,6 +43,7 @@ fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { ); } +#[expect(clippy::expect_used)] fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { let content = request_body["content"][0]["text"] .as_str() @@ -54,10 +55,61 @@ fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { ); } +/// Writes an `auth.json` into the provided `codex_home` with the specified parameters. +/// Returns the fake JWT string written to `tokens.id_token`. +#[expect(clippy::unwrap_used)] +fn write_auth_json( + codex_home: &TempDir, + openai_api_key: Option<&str>, + chatgpt_plan_type: &str, + access_token: &str, + account_id: Option<&str>, +) -> String { + use base64::Engine as _; + use serde_json::json; + + let header = json!({ "alg": "none", "typ": "JWT" }); + let payload = json!({ + "email": "user@example.com", + "https://api.openai.com/auth": { + "chatgpt_plan_type": chatgpt_plan_type, + "chatgpt_account_id": account_id.unwrap_or("acc-123") + } + }); + + let b64 = |b: &[u8]| base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b); + let header_b64 = b64(&serde_json::to_vec(&header).unwrap()); + let payload_b64 = b64(&serde_json::to_vec(&payload).unwrap()); + let signature_b64 = b64(b"sig"); + let fake_jwt = format!("{header_b64}.{payload_b64}.{signature_b64}"); + + let mut tokens = json!({ + "id_token": fake_jwt, + "access_token": access_token, + "refresh_token": "refresh-test", + }); + if let Some(acc) = account_id { + tokens["account_id"] = json!(acc); + } + + let auth_json = json!({ + "OPENAI_API_KEY": openai_api_key, + "tokens": tokens, + // RFC3339 datetime; value doesn't matter for these tests + "last_refresh": "2025-08-06T20:41:36.232376Z", + }); + + std::fs::write( + codex_home.path().join("auth.json"), + serde_json::to_string_pretty(&auth_json).unwrap(), + ) + .unwrap(); + + fake_jwt +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_session_id_and_model_headers_in_request() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -90,14 +142,15 @@ async fn includes_session_id_and_model_headers_in_request() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + conversation_id, + session_configured: _, + } = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation"); codex .submit(Op::UserInput { @@ -108,13 +161,6 @@ async fn includes_session_id_and_model_headers_in_request() { .await .unwrap(); - let EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, .. }) = - wait_for_event(&codex, |ev| matches!(ev, EventMsg::SessionConfigured(_))).await - else { - unreachable!() - }; - - let current_session_id = Some(session_id.to_string()); wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; // get request from the server @@ -123,10 +169,9 @@ async fn includes_session_id_and_model_headers_in_request() { let request_authorization = request.headers.get("authorization").unwrap(); let request_originator = request.headers.get("originator").unwrap(); - assert!(current_session_id.is_some()); assert_eq!( request_session_id.to_str().unwrap(), - current_session_id.as_ref().unwrap() + conversation_id.to_string() ); assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); assert_eq!( @@ -137,8 +182,6 @@ async fn includes_session_id_and_model_headers_in_request() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_base_instructions_override_in_request() { - #![allow(clippy::unwrap_used)] - // Mock server let server = MockServer::start().await; @@ -164,14 +207,12 @@ async fn includes_base_instructions_override_in_request() { config.base_instructions = Some("test instructions".to_string()); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -197,8 +238,6 @@ async fn includes_base_instructions_override_in_request() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn originator_config_override_is_used() { - #![allow(clippy::unwrap_used)] - // Mock server let server = MockServer::start().await; @@ -221,16 +260,14 @@ async fn originator_config_override_is_used() { let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - config.internal_originator = Some("my_override".to_string()); + config.responses_originator_header = "my_override".to_owned(); - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -250,8 +287,6 @@ async fn originator_config_override_is_used() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn chatgpt_auth_sends_correct_request() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -283,11 +318,15 @@ async fn chatgpt_auth_sends_correct_request() { let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = - Codex::spawn(config, Some(create_dummy_codex_auth()), ctrl_c.clone()) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + conversation_id, + session_configured: _, + } = conversation_manager + .new_conversation_with_auth(config, Some(create_dummy_codex_auth())) + .await + .expect("create new conversation"); codex .submit(Op::UserInput { @@ -298,13 +337,6 @@ async fn chatgpt_auth_sends_correct_request() { .await .unwrap(); - let EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, .. }) = - wait_for_event(&codex, |ev| matches!(ev, EventMsg::SessionConfigured(_))).await - else { - unreachable!() - }; - - let current_session_id = Some(session_id.to_string()); wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; // get request from the server @@ -315,10 +347,9 @@ async fn chatgpt_auth_sends_correct_request() { let request_chatgpt_account_id = request.headers.get("chatgpt-account-id").unwrap(); let request_body = request.body_json::().unwrap(); - assert!(current_session_id.is_some()); assert_eq!( request_session_id.to_str().unwrap(), - current_session_id.as_ref().unwrap() + conversation_id.to_string() ); assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); assert_eq!( @@ -335,9 +366,157 @@ async fn chatgpt_auth_sends_correct_request() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn includes_user_instructions_message_in_request() { - #![allow(clippy::unwrap_used)] +async fn prefers_chatgpt_token_when_config_prefers_chatgpt() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + // Mock server + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect ChatGPT base path and correct headers + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(header_regex("Authorization", r"Bearer Access-123")) + .and(header_regex("chatgpt-account-id", r"acc-123")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + // Write auth.json that contains both API key and ChatGPT tokens for a plan that should prefer ChatGPT. + let _jwt = write_auth_json( + &codex_home, + Some("sk-test-key"), + "pro", + "Access-123", + Some("acc-123"), + ); + + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.preferred_auth_method = AuthMode::ChatGPT; + + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + .. + } = conversation_manager + .new_conversation(config) + .await + .expect("create new conversation"); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // verify request body flags + let request = &server.received_requests().await.unwrap()[0]; + let request_body = request.body_json::().unwrap(); + assert!( + !request_body["store"].as_bool().unwrap(), + "store should be false for ChatGPT auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + // Mock server + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect API key header, no ChatGPT account header required. + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(header_regex("Authorization", r"Bearer sk-test-key")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + // Write auth.json that contains both API key and ChatGPT tokens for a plan that should prefer ChatGPT, + // but config will force API key preference. + let _jwt = write_auth_json( + &codex_home, + Some("sk-test-key"), + "pro", + "Access-123", + Some("acc-123"), + ); + + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.preferred_auth_method = AuthMode::ApiKey; + + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + .. + } = conversation_manager + .new_conversation(config) + .await + .expect("create new conversation"); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // verify request body flags + let request = &server.received_requests().await.unwrap()[0]; + let request_body = request.body_json::().unwrap(); + assert!( + request_body["store"].as_bool().unwrap(), + "store should be true for API key auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn includes_user_instructions_message_in_request() { let server = MockServer::start().await; let first = ResponseTemplate::new(200) @@ -361,14 +540,12 @@ async fn includes_user_instructions_message_in_request() { config.model_provider = model_provider; config.user_instructions = Some("be nice".to_string()); - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -391,17 +568,15 @@ async fn includes_user_instructions_message_in_request() { .contains("be nice") ); assert_message_role(&request_body["input"][0], "user"); - assert_message_starts_with(&request_body["input"][0], "\n\n"); - assert_message_ends_with(&request_body["input"][0], ""); + assert_message_starts_with(&request_body["input"][0], ""); + assert_message_ends_with(&request_body["input"][0], ""); assert_message_role(&request_body["input"][1], "user"); - assert_message_starts_with(&request_body["input"][1], "\n\n"); - assert_message_ends_with(&request_body["input"][1], ""); + assert_message_starts_with(&request_body["input"][1], ""); + assert_message_ends_with(&request_body["input"][1], ""); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn azure_overrides_assign_properties_used_for_responses_url() { - #![allow(clippy::unwrap_used)] - let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; // Mock server @@ -457,8 +632,12 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, None, ctrl_c.clone()).await.unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, None) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -474,8 +653,6 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn env_var_overrides_loaded_auth() { - #![allow(clippy::unwrap_used)] - let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; // Mock server @@ -531,11 +708,12 @@ async fn env_var_overrides_loaded_auth() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = - Codex::spawn(config, Some(create_dummy_codex_auth()), ctrl_c.clone()) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(create_dummy_codex_auth())) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/core/tests/common/lib.rs b/codex-rs/core/tests/common/lib.rs index 18bae310be9..244d093e7de 100644 --- a/codex-rs/core/tests/common/lib.rs +++ b/codex-rs/core/tests/common/lib.rs @@ -1,7 +1,8 @@ -#![allow(clippy::expect_used)] +#![expect(clippy::expect_used)] use tempfile::TempDir; +use codex_core::CodexConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::ConfigToml; @@ -46,6 +47,26 @@ pub fn load_sse_fixture(path: impl AsRef) -> String { .collect() } +pub fn load_sse_fixture_with_id_from_str(raw: &str, id: &str) -> String { + let replaced = raw.replace("__ID__", id); + let events: Vec = + serde_json::from_str(&replaced).expect("parse JSON fixture"); + events + .into_iter() + .map(|e| { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .expect("fixture event missing type"); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + format!("event: {kind}\n\n") + } else { + format!("event: {kind}\ndata: {e}\n\n") + } + }) + .collect() +} + /// Same as [`load_sse_fixture`], but replaces the placeholder `__ID__` in the /// fixture template with the supplied identifier before parsing. This lets a /// single JSON template be reused by multiple tests that each need a unique @@ -72,7 +93,7 @@ pub fn load_sse_fixture_with_id(path: impl AsRef, id: &str) -> } pub async fn wait_for_event( - codex: &codex_core::Codex, + codex: &CodexConversation, predicate: F, ) -> codex_core::protocol::EventMsg where @@ -83,7 +104,7 @@ where } pub async fn wait_for_event_with_timeout( - codex: &codex_core::Codex, + codex: &CodexConversation, mut predicate: F, wait_time: tokio::time::Duration, ) -> codex_core::protocol::EventMsg diff --git a/codex-rs/core/tests/compact.rs b/codex-rs/core/tests/compact.rs index fa5c81d883a..28b1ca8d74d 100644 --- a/codex-rs/core/tests/compact.rs +++ b/codex-rs/core/tests/compact.rs @@ -1,7 +1,6 @@ #![expect(clippy::unwrap_used)] -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; use codex_core::protocol::EventMsg; @@ -142,14 +141,12 @@ async fn summarize_context_three_requests_and_instructions() { let home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("dummy")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("dummy"))) + .await + .unwrap() + .conversation; // 1) Normal user input – should hit server once. codex diff --git a/codex-rs/core/tests/exec.rs b/codex-rs/core/tests/exec.rs index 9bead7ef60d..f011aa7a0ac 100644 --- a/codex-rs/core/tests/exec.rs +++ b/codex-rs/core/tests/exec.rs @@ -1,8 +1,6 @@ #![cfg(target_os = "macos")] -#![expect(clippy::unwrap_used, clippy::expect_used)] use std::collections::HashMap; -use std::sync::Arc; use codex_core::exec::ExecParams; use codex_core::exec::ExecToolCallOutput; @@ -11,7 +9,6 @@ use codex_core::exec::process_exec_tool_call; use codex_core::protocol::SandboxPolicy; use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; use tempfile::TempDir; -use tokio::sync::Notify; use codex_core::error::Result; @@ -26,6 +23,7 @@ fn skip_test() -> bool { false } +#[expect(clippy::expect_used)] async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result { let sandbox_type = get_platform_sandbox().expect("should be able to get sandbox type"); assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); @@ -39,10 +37,9 @@ async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result) -> Vec { let mut out = Vec::new(); @@ -57,13 +55,11 @@ async fn test_exec_stdout_stream_events_echo() { justification: None, }; - let ctrl_c = Arc::new(Notify::new()); let policy = SandboxPolicy::new_read_only_policy(); let result = process_exec_tool_call( params, SandboxType::None, - ctrl_c, &policy, &None, Some(stdout_stream), @@ -109,13 +105,11 @@ async fn test_exec_stderr_stream_events_echo() { justification: None, }; - let ctrl_c = Arc::new(Notify::new()); let policy = SandboxPolicy::new_read_only_policy(); let result = process_exec_tool_call( params, SandboxType::None, - ctrl_c, &policy, &None, Some(stdout_stream), diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs deleted file mode 100644 index 81b3bb2a12f..00000000000 --- a/codex-rs/core/tests/live_agent.rs +++ /dev/null @@ -1,209 +0,0 @@ -#![expect(clippy::unwrap_used, clippy::expect_used)] - -//! Live integration tests that exercise the full [`Agent`] stack **against the real -//! OpenAI `/v1/responses` API**. These tests complement the lightweight mock‑based -//! unit tests by verifying that the agent can drive an end‑to‑end conversation, -//! stream incremental events, execute function‑call tool invocations and safely -//! chain multiple turns inside a single session – the exact scenarios that have -//! historically been brittle. -//! -//! The live tests are **ignored by default** so CI remains deterministic and free -//! of external dependencies. Developers can opt‑in locally with e.g. -//! -//! ```bash -//! OPENAI_API_KEY=sk‑... cargo test --test live_agent -- --ignored --nocapture -//! ``` -//! -//! Make sure your key has access to the experimental *Responses* API and that -//! any billable usage is acceptable. - -use std::time::Duration; - -use codex_core::Codex; -use codex_core::CodexSpawnOk; -use codex_core::error::CodexErr; -use codex_core::protocol::AgentMessageEvent; -use codex_core::protocol::ErrorEvent; -use codex_core::protocol::EventMsg; -use codex_core::protocol::InputItem; -use codex_core::protocol::Op; -use core_test_support::load_default_config_for_test; -use tempfile::TempDir; -use tokio::sync::Notify; -use tokio::time::timeout; - -fn api_key_available() -> bool { - std::env::var("OPENAI_API_KEY").is_ok() -} - -/// Helper that spawns a fresh Agent and sends the mandatory *ConfigureSession* -/// submission. The caller receives the constructed [`Agent`] plus the unique -/// submission id used for the initialization message. -async fn spawn_codex() -> Result { - assert!( - api_key_available(), - "OPENAI_API_KEY must be set for live tests" - ); - - let codex_home = TempDir::new().unwrap(); - let mut config = load_default_config_for_test(&codex_home); - config.model_provider.request_max_retries = Some(2); - config.model_provider.stream_max_retries = Some(2); - let CodexSpawnOk { codex: agent, .. } = - Codex::spawn(config, None, std::sync::Arc::new(Notify::new())).await?; - - Ok(agent) -} - -/// Verifies that the agent streams incremental *AgentMessage* events **before** -/// emitting `TaskComplete` and that a second task inside the same session does -/// not get tripped up by a stale `previous_response_id`. -#[ignore] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn live_streaming_and_prev_id_reset() { - if !api_key_available() { - eprintln!("skipping live_streaming_and_prev_id_reset – OPENAI_API_KEY not set"); - return; - } - - let codex = spawn_codex().await.unwrap(); - - // ---------- Task 1 ---------- - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: "Say the words 'stream test'".into(), - }], - }) - .await - .unwrap(); - - let mut saw_message_before_complete = false; - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for task1 events") - .expect("agent closed"); - - match ev.msg { - EventMsg::AgentMessage(_) => saw_message_before_complete = true, - EventMsg::TaskComplete(_) => break, - EventMsg::Error(ErrorEvent { message }) => { - panic!("agent reported error in task1: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!( - saw_message_before_complete, - "Agent did not stream any AgentMessage before TaskComplete" - ); - - // ---------- Task 2 (same session) ---------- - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: "Respond with exactly: second turn succeeded".into(), - }], - }) - .await - .unwrap(); - - let mut got_expected = false; - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for task2 events") - .expect("agent closed"); - - match &ev.msg { - EventMsg::AgentMessage(AgentMessageEvent { message }) - if message.contains("second turn succeeded") => - { - got_expected = true; - } - EventMsg::TaskComplete(_) => break, - EventMsg::Error(ErrorEvent { message }) => { - panic!("agent reported error in task2: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!(got_expected, "second task did not receive expected answer"); -} - -/// Exercises a *function‑call → shell execution* round‑trip by instructing the -/// model to run a harmless `echo` command. The test asserts that: -/// 1. the function call is executed (we see `ExecCommandBegin`/`End` events) -/// 2. the captured stdout reaches the client unchanged. -#[ignore] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn live_shell_function_call() { - if !api_key_available() { - eprintln!("skipping live_shell_function_call – OPENAI_API_KEY not set"); - return; - } - - let codex = spawn_codex().await.unwrap(); - - const MARKER: &str = "codex_live_echo_ok"; - - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: format!( - "Use the shell function to run the command `echo {MARKER}` and no other commands." - ), - }], - }) - .await - .unwrap(); - - let mut saw_begin = false; - let mut saw_end_with_output = false; - - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for function‑call events") - .expect("agent closed"); - - match ev.msg { - EventMsg::ExecCommandBegin(codex_core::protocol::ExecCommandBeginEvent { - command, - .. - }) => { - assert_eq!(command, vec!["echo", MARKER]); - saw_begin = true; - } - EventMsg::ExecCommandEnd(codex_core::protocol::ExecCommandEndEvent { - stdout, - exit_code, - .. - }) => { - assert_eq!(exit_code, 0, "echo returned non‑zero exit code"); - assert!(stdout.contains(MARKER)); - saw_end_with_output = true; - } - EventMsg::TaskComplete(_) => break, - EventMsg::Error(codex_core::protocol::ErrorEvent { message }) => { - panic!("agent error during shell test: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!(saw_begin, "ExecCommandBegin event missing"); - assert!( - saw_end_with_output, - "ExecCommandEnd with expected output missing" - ); -} diff --git a/codex-rs/core/tests/live_cli.rs b/codex-rs/core/tests/live_cli.rs index d79e242c4d8..e77fe6998e1 100644 --- a/codex-rs/core/tests/live_cli.rs +++ b/codex-rs/core/tests/live_cli.rs @@ -17,7 +17,7 @@ fn require_api_key() -> String { /// Helper that spawns the binary inside a TempDir with minimal flags. Returns (Assert, TempDir). fn run_live(prompt: &str) -> (assert_cmd::assert::Assert, TempDir) { - #![allow(clippy::unwrap_used)] + #![expect(clippy::unwrap_used)] use std::io::Read; use std::io::Write; use std::thread; @@ -113,7 +113,6 @@ fn run_live(prompt: &str) -> (assert_cmd::assert::Assert, TempDir) { #[ignore] #[test] fn live_create_file_hello_txt() { - #![allow(clippy::unwrap_used)] if std::env::var("OPENAI_API_KEY").is_err() { eprintln!("skipping live_create_file_hello_txt – OPENAI_API_KEY not set"); return; diff --git a/codex-rs/core/tests/prompt_caching.rs b/codex-rs/core/tests/prompt_caching.rs index f460fc30042..bf49262a88a 100644 --- a/codex-rs/core/tests/prompt_caching.rs +++ b/codex-rs/core/tests/prompt_caching.rs @@ -1,12 +1,14 @@ -#![allow(clippy::expect_used, clippy::unwrap_used)] - -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; +use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; +use codex_core::protocol::SandboxPolicy; +use codex_core::protocol_config_types::ReasoningEffort; +use codex_core::protocol_config_types::ReasoningSummary; +use codex_core::shell::default_user_shell; use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id; @@ -25,7 +27,6 @@ fn sse_completed(id: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn prefixes_context_and_instructions_once_and_consistently_across_requests() { - #![allow(clippy::unwrap_used)] use pretty_assertions::assert_eq; let server = MockServer::start().await; @@ -55,14 +56,12 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests config.model_provider = model_provider; config.user_instructions = Some("be consistent and helpful".to_string()); - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -87,9 +86,20 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests let requests = server.received_requests().await.unwrap(); assert_eq!(requests.len(), 2, "expected two POST requests"); + let shell = default_user_shell().await; + let expected_env_text = format!( - "\n\nCurrent working directory: {}\nApproval policy: on-request\nSandbox policy: read-only\nNetwork access: restricted\n\n\n", - cwd.path().to_string_lossy() + r#" + {} + on-request + read-only + restricted +{}"#, + cwd.path().to_string_lossy(), + match shell.name() { + Some(name) => format!(" {}\n", name), + None => String::new(), + } ); let expected_ui_text = "\n\nbe consistent and helpful\n\n"; @@ -116,7 +126,7 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests let body1 = requests[0].body_json::().unwrap(); assert_eq!( body1["input"], - serde_json::json!([expected_env_msg, expected_ui_msg, expected_user_message_1]) + serde_json::json!([expected_ui_msg, expected_env_msg, expected_user_message_1]) ); let expected_user_message_2 = serde_json::json!({ @@ -135,3 +145,229 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests ); assert_eq!(body2["input"], expected_body2); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() { + use pretty_assertions::assert_eq; + + let server = MockServer::start().await; + + let sse = sse_completed("resp"); + let template = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"); + + // Expect two POSTs to /v1/responses + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(template) + .expect(2) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let cwd = TempDir::new().unwrap(); + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.cwd = cwd.path().to_path_buf(); + config.model_provider = model_provider; + config.user_instructions = Some("be consistent and helpful".to_string()); + + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; + + // First turn + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 1".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + let writable = TempDir::new().unwrap(); + codex + .submit(Op::OverrideTurnContext { + cwd: None, + approval_policy: Some(AskForApproval::Never), + sandbox_policy: Some(SandboxPolicy::WorkspaceWrite { + writable_roots: vec![writable.path().to_path_buf()], + network_access: true, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }), + model: Some("o3".to_string()), + effort: Some(ReasoningEffort::High), + summary: Some(ReasoningSummary::Detailed), + }) + .await + .unwrap(); + + // Second turn after overrides + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 2".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Verify we issued exactly two requests, and the cached prefix stayed identical. + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2, "expected two POST requests"); + + let body1 = requests[0].body_json::().unwrap(); + let body2 = requests[1].body_json::().unwrap(); + // prompt_cache_key should remain constant across overrides + assert_eq!( + body1["prompt_cache_key"], body2["prompt_cache_key"], + "prompt_cache_key should not change across overrides" + ); + + // The entire prefix from the first request should be identical and reused + // as the prefix of the second request, ensuring cache hit potential. + let expected_user_message_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 2" } ] + }); + // After overriding the turn context, the environment context should be emitted again + // reflecting the new approval policy and sandbox settings. Omit cwd because it did + // not change. + let expected_env_text_2 = r#" + never + workspace-write + enabled +"#; + let expected_env_msg_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": expected_env_text_2 } ] + }); + let expected_body2 = serde_json::json!( + [ + body1["input"].as_array().unwrap().as_slice(), + [expected_env_msg_2, expected_user_message_2].as_slice(), + ] + .concat() + ); + assert_eq!(body2["input"], expected_body2); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn per_turn_overrides_keep_cached_prefix_and_key_constant() { + use pretty_assertions::assert_eq; + + let server = MockServer::start().await; + + let sse = sse_completed("resp"); + let template = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"); + + // Expect two POSTs to /v1/responses + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(template) + .expect(2) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let cwd = TempDir::new().unwrap(); + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.cwd = cwd.path().to_path_buf(); + config.model_provider = model_provider; + config.user_instructions = Some("be consistent and helpful".to_string()); + + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; + + // First turn + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 1".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Second turn using per-turn overrides via UserTurn + let new_cwd = TempDir::new().unwrap(); + let writable = TempDir::new().unwrap(); + codex + .submit(Op::UserTurn { + items: vec![InputItem::Text { + text: "hello 2".into(), + }], + cwd: new_cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::WorkspaceWrite { + writable_roots: vec![writable.path().to_path_buf()], + network_access: true, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }, + model: "o3".to_string(), + effort: ReasoningEffort::High, + summary: ReasoningSummary::Detailed, + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Verify we issued exactly two requests, and the cached prefix stayed identical. + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2, "expected two POST requests"); + + let body1 = requests[0].body_json::().unwrap(); + let body2 = requests[1].body_json::().unwrap(); + + // prompt_cache_key should remain constant across per-turn overrides + assert_eq!( + body1["prompt_cache_key"], body2["prompt_cache_key"], + "prompt_cache_key should not change across per-turn overrides" + ); + + // The entire prefix from the first request should be identical and reused + // as the prefix of the second request. + let expected_user_message_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 2" } ] + }); + let expected_body2 = serde_json::json!( + [ + body1["input"].as_array().unwrap().as_slice(), + [expected_user_message_2].as_slice(), + ] + .concat() + ); + assert_eq!(body2["input"], expected_body2); +} diff --git a/codex-rs/core/tests/sandbox.rs b/codex-rs/core/tests/seatbelt.rs similarity index 96% rename from codex-rs/core/tests/sandbox.rs rename to codex-rs/core/tests/seatbelt.rs index ae5bdc44a6b..638ac48bb87 100644 --- a/codex-rs/core/tests/sandbox.rs +++ b/codex-rs/core/tests/seatbelt.rs @@ -1,5 +1,7 @@ #![cfg(target_os = "macos")] -#![expect(clippy::expect_used)] + +//! Tests for the macOS sandboxing that are specific to Seatbelt. +//! Tests that apply to both Mac and Linux sandboxing should go in sandbox.rs. use std::collections::HashMap; use std::path::Path; @@ -157,6 +159,7 @@ async fn read_only_forbids_all_writes() { .await; } +#[expect(clippy::expect_used)] fn create_test_scenario(tmp: &TempDir) -> TestScenario { let repo_parent = tmp.path().to_path_buf(); let repo_root = repo_parent.join("repo"); @@ -174,6 +177,7 @@ fn create_test_scenario(tmp: &TempDir) -> TestScenario { } } +#[expect(clippy::expect_used)] /// Note that `path` must be absolute. async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { assert!(path.is_absolute(), "Path must be absolute: {path:?}"); diff --git a/codex-rs/core/tests/stream_error_allows_next_turn.rs b/codex-rs/core/tests/stream_error_allows_next_turn.rs index 1500c789e97..415e75a4264 100644 --- a/codex-rs/core/tests/stream_error_allows_next_turn.rs +++ b/codex-rs/core/tests/stream_error_allows_next_turn.rs @@ -1,7 +1,6 @@ use std::time::Duration; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::WireApi; use codex_core::protocol::EventMsg; @@ -26,7 +25,6 @@ fn sse_completed(id: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn continue_after_stream_error() { - #![allow(clippy::unwrap_used)] if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -90,13 +88,12 @@ async fn continue_after_stream_error() { config.base_instructions = Some("You are a helpful assistant".to_string()); config.model_provider = provider; - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - std::sync::Arc::new(tokio::sync::Notify::new()), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .unwrap() + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index 0ded3337abe..3fb3f642d7c 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -3,8 +3,7 @@ use std::time::Duration; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; @@ -34,8 +33,6 @@ fn sse_completed(id: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn retries_on_early_close() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -93,17 +90,15 @@ async fn retries_on_early_close() { requires_openai_auth: false, }; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c, - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .unwrap() + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index aee480d7b45..89dc3951e7c 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -26,6 +26,7 @@ codex-common = { path = "../common", features = [ ] } codex-core = { path = "../core" } codex-ollama = { path = "../ollama" } +codex-protocol = { path = "../protocol" } owo-colors = "4.2.0" serde_json = "1" shlex = "1.3.0" @@ -41,5 +42,8 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } [dev-dependencies] assert_cmd = "2" +core_test_support = { path = "../core/tests/common" } +libc = "0.2" predicates = "3" tempfile = "3.13.0" +wiremock = "0.6" diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index b7b3c27dc54..1ba10c34e2e 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -29,9 +29,9 @@ pub(crate) fn handle_last_message(last_agent_message: Option<&str>, output_file: } fn write_last_message_file(contents: &str, last_message_path: Option<&Path>) { - if let Some(path) = last_message_path { - if let Err(e) = std::fs::write(path, contents) { - eprintln!("Failed to write last message file {path:?}: {e}"); - } + if let Some(path) = last_message_path + && let Err(e) = std::fs::write(path, contents) + { + eprintln!("Failed to write last message file {path:?}: {e}"); } } diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 22cf1304620..9a562cbd4da 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -20,7 +20,9 @@ use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::PatchApplyBeginEvent; use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; +use codex_core::protocol::StreamErrorEvent; use codex_core::protocol::TaskCompleteEvent; +use codex_core::protocol::TurnAbortReason; use codex_core::protocol::TurnDiffEvent; use owo_colors::OwoColorize; use owo_colors::Style; @@ -173,6 +175,9 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { ts_println!(self, "{}", message.style(self.dimmed)); } + EventMsg::StreamError(StreamErrorEvent { message }) => { + ts_println!(self, "{}", message.style(self.dimmed)); + } EventMsg::TaskStarted => { // Ignore. } @@ -191,7 +196,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.answer_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }) => { @@ -207,7 +212,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.reasoning_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentReasoningSectionBreak(_) => { @@ -215,7 +220,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { return CodexStatus::Running; } println!(); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { text }) => { @@ -224,7 +229,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { } if !self.raw_reasoning_started { print!("{text}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } else { println!(); @@ -241,7 +246,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.raw_reasoning_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentMessage(AgentMessageEvent { message }) => { @@ -522,6 +527,17 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::GetHistoryEntryResponse(_) => { // Currently ignored in exec output. } + EventMsg::McpListToolsResponse(_) => { + // Currently ignored in exec output. + } + EventMsg::TurnAborted(abort_reason) => match abort_reason.reason { + TurnAbortReason::Interrupted => { + ts_println!(self, "task interrupted"); + } + TurnAbortReason::Replaced => { + ts_println!(self, "task aborted: replaced by a new task"); + } + }, EventMsg::ShutdownComplete => return CodexStatus::Shutdown, } CodexStatus::Running diff --git a/codex-rs/exec/src/event_processor_with_json_output.rs b/codex-rs/exec/src/event_processor_with_json_output.rs index 76985518e65..11e9732e99c 100644 --- a/codex-rs/exec/src/event_processor_with_json_output.rs +++ b/codex-rs/exec/src/event_processor_with_json_output.rs @@ -28,7 +28,7 @@ impl EventProcessor for EventProcessorWithJsonOutput { .into_iter() .map(|(key, value)| (key.to_string(), value)) .collect::>(); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] let config_json = serde_json::to_string(&entries).expect("Failed to serialize config summary to JSON"); println!("{config_json}"); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 6ed57898b2d..e18314fbc7b 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -6,15 +6,13 @@ mod event_processor_with_json_output; use std::io::IsTerminal; use std::io::Read; use std::path::PathBuf; -use std::sync::Arc; pub use cli::Cli; use codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID; -use codex_core::codex_wrapper::CodexConversation; -use codex_core::codex_wrapper::{self}; +use codex_core::ConversationManager; +use codex_core::NewConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; -use codex_core::config_types::SandboxMode; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; @@ -23,6 +21,7 @@ use codex_core::protocol::Op; use codex_core::protocol::TaskCompleteEvent; use codex_core::util::is_inside_git_repo; use codex_ollama::DEFAULT_OSS_MODEL; +use codex_protocol::config_types::SandboxMode; use event_processor_with_human_output::EventProcessorWithHumanOutput; use event_processor_with_json_output::EventProcessorWithJsonOutput; use tracing::debug; @@ -147,6 +146,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: None, + include_apply_patch_tool: None, disable_response_storage: oss.then_some(true), show_raw_agent_reasoning: oss.then_some(true), }; @@ -185,35 +185,30 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any std::process::exit(1); } - let CodexConversation { - codex: codex_wrapper, + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation_id: _, + conversation, session_configured, - ctrl_c, - .. - } = codex_wrapper::init_codex(config).await?; - let codex = Arc::new(codex_wrapper); + } = conversation_manager.new_conversation(config).await?; info!("Codex initialized with event: {session_configured:?}"); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); { - let codex = codex.clone(); + let conversation = conversation.clone(); tokio::spawn(async move { loop { - let interrupted = ctrl_c.notified(); tokio::select! { - _ = interrupted => { - // Forward an interrupt to the codex so it can abort any in‑flight task. - let _ = codex - .submit( - Op::Interrupt, - ) - .await; + _ = tokio::signal::ctrl_c() => { + tracing::debug!("Keyboard interrupt"); + // Immediately notify Codex to abort any in‑flight task. + conversation.submit(Op::Interrupt).await.ok(); - // Exit the inner loop and return to the main input prompt. The codex + // Exit the inner loop and return to the main input prompt. The codex // will emit a `TurnInterrupted` (Error) event which is drained later. break; } - res = codex.next_event() => match res { + res = conversation.next_event() => match res { Ok(event) => { debug!("Received event: {event:?}"); @@ -243,9 +238,9 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any .into_iter() .map(|path| InputItem::LocalImage { path }) .collect(); - let initial_images_event_id = codex.submit(Op::UserInput { items }).await?; + let initial_images_event_id = conversation.submit(Op::UserInput { items }).await?; info!("Sent images with event ID: {initial_images_event_id}"); - while let Ok(event) = codex.next_event().await { + while let Ok(event) = conversation.next_event().await { if event.id == initial_images_event_id && matches!( event.msg, @@ -261,7 +256,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any // Send the prompt. let items: Vec = vec![InputItem::Text { text: prompt }]; - let initial_prompt_task_id = codex.submit(Op::UserInput { items }).await?; + let initial_prompt_task_id = conversation.submit(Op::UserInput { items }).await?; info!("Sent prompt with event ID: {initial_prompt_task_id}"); // Run the loop until the task is complete. @@ -270,7 +265,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any match shutdown { CodexStatus::Running => continue, CodexStatus::InitiateShutdown => { - codex.submit(Op::Shutdown).await?; + conversation.submit(Op::Shutdown).await?; } CodexStatus::Shutdown => { break; diff --git a/codex-rs/exec/tests/apply_patch.rs b/codex-rs/exec/tests/apply_patch.rs index f65d32e1c8f..ecce43d7325 100644 --- a/codex-rs/exec/tests/apply_patch.rs +++ b/codex-rs/exec/tests/apply_patch.rs @@ -1,3 +1,5 @@ +#![allow(clippy::expect_used, clippy::unwrap_used)] + use anyhow::Context; use assert_cmd::prelude::*; use codex_core::CODEX_APPLY_PATCH_ARG1; @@ -37,3 +39,152 @@ fn test_standalone_exec_cli_can_use_apply_patch() -> anyhow::Result<()> { ); Ok(()) } + +#[cfg(not(target_os = "windows"))] +#[tokio::test] +async fn test_apply_patch_tool() -> anyhow::Result<()> { + use core_test_support::load_sse_fixture_with_id_from_str; + use tempfile::TempDir; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::method; + use wiremock::matchers::path; + + const SSE_TOOL_CALL_ADD: &str = r#"[ + { + "type": "response.output_item.done", + "item": { + "type": "function_call", + "name": "apply_patch", + "arguments": "{\n \"input\": \"*** Begin Patch\\n*** Add File: test.md\\n+Hello world\\n*** End Patch\"\n}", + "call_id": "__ID__" + } + }, + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + const SSE_TOOL_CALL_UPDATE: &str = r#"[ + { + "type": "response.output_item.done", + "item": { + "type": "function_call", + "name": "apply_patch", + "arguments": "{\n \"input\": \"*** Begin Patch\\n*** Update File: test.md\\n@@\\n-Hello world\\n+Final text\\n*** End Patch\"\n}", + "call_id": "__ID__" + } + }, + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + const SSE_TOOL_CALL_COMPLETED: &str = r#"[ + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + // Start a mock model server + let server = MockServer::start().await; + + // First response: model calls apply_patch to create test.md + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_ADD, "call1"), + "text/event-stream", + ); + + Mock::given(method("POST")) + // .and(path("/v1/responses")) + .respond_with(first) + .up_to_n_times(1) + .mount(&server) + .await; + + // Second response: model calls apply_patch to update test.md + let second = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_UPDATE, "call2"), + "text/event-stream", + ); + + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(second) + .up_to_n_times(1) + .mount(&server) + .await; + + let final_completed = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_COMPLETED, "resp3"), + "text/event-stream", + ); + + Mock::given(method("POST")) + // .and(path("/v1/responses")) + .respond_with(final_completed) + .expect(1) + .mount(&server) + .await; + + let tmp_cwd = TempDir::new().unwrap(); + Command::cargo_bin("codex-exec") + .context("should find binary for codex-exec")? + .current_dir(tmp_cwd.path()) + .env("CODEX_HOME", tmp_cwd.path()) + .env("OPENAI_API_KEY", "dummy") + .env("OPENAI_BASE_URL", format!("{}/v1", server.uri())) + .arg("--skip-git-repo-check") + .arg("-s") + .arg("workspace-write") + .arg("foo") + .assert() + .success(); + + // Verify final file contents + let final_path = tmp_cwd.path().join("test.md"); + let contents = std::fs::read_to_string(&final_path) + .unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display())); + assert_eq!(contents, "Final text\n"); + Ok(()) +} diff --git a/codex-rs/exec/tests/sandbox.rs b/codex-rs/exec/tests/sandbox.rs new file mode 100644 index 00000000000..a98e0a97607 --- /dev/null +++ b/codex-rs/exec/tests/sandbox.rs @@ -0,0 +1,219 @@ +#![cfg(unix)] +use codex_core::protocol::SandboxPolicy; +use codex_core::spawn::StdioPolicy; +use std::collections::HashMap; +use std::future::Future; +use std::io; +use std::path::PathBuf; +use std::process::ExitStatus; +use tokio::process::Child; + +#[cfg(target_os = "macos")] +async fn spawn_command_under_sandbox( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + use codex_core::seatbelt::spawn_command_under_seatbelt; + spawn_command_under_seatbelt(command, sandbox_policy, cwd, stdio_policy, env).await +} + +#[cfg(target_os = "linux")] +async fn spawn_command_under_sandbox( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + use codex_core::landlock::spawn_command_under_linux_sandbox; + let codex_linux_sandbox_exe = assert_cmd::cargo::cargo_bin("codex-exec"); + spawn_command_under_linux_sandbox( + codex_linux_sandbox_exe, + command, + sandbox_policy, + cwd, + stdio_policy, + env, + ) + .await +} + +#[tokio::test] +async fn python_multiprocessing_lock_works_under_sandbox() { + #[cfg(target_os = "macos")] + let writable_roots = Vec::::new(); + + // From https://man7.org/linux/man-pages/man7/sem_overview.7.html + // + // > On Linux, named semaphores are created in a virtual filesystem, + // > normally mounted under /dev/shm. + #[cfg(target_os = "linux")] + let writable_roots = vec![PathBuf::from("/dev/shm")]; + + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots, + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + + let python_code = r#"import multiprocessing +from multiprocessing import Lock, Process + +def f(lock): + with lock: + print("Lock acquired in child process") + +if __name__ == '__main__': + lock = Lock() + p = Process(target=f, args=(lock,)) + p.start() + p.join() +"#; + + let mut child = spawn_command_under_sandbox( + vec![ + "python3".to_string(), + "-c".to_string(), + python_code.to_string(), + ], + &policy, + std::env::current_dir().expect("should be able to get current dir"), + StdioPolicy::Inherit, + HashMap::new(), + ) + .await + .expect("should be able to spawn python under sandbox"); + + let status = child.wait().await.expect("should wait for child process"); + assert!(status.success(), "python exited with {status:?}"); +} + +fn unix_sock_body() { + unsafe { + let mut fds = [0i32; 2]; + let r = libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()); + assert_eq!( + r, + 0, + "socketpair(AF_UNIX, SOCK_DGRAM) failed: {}", + io::Error::last_os_error() + ); + + let msg = b"hello_unix"; + // write() from one end (generic write is allowed) + let sent = libc::write(fds[0], msg.as_ptr() as *const libc::c_void, msg.len()); + assert!(sent >= 0, "write() failed: {}", io::Error::last_os_error()); + + // recvfrom() on the other end. We don’t need the address for socketpair, + // so we pass null pointers for src address. + let mut buf = [0u8; 64]; + let recvd = libc::recvfrom( + fds[1], + buf.as_mut_ptr() as *mut libc::c_void, + buf.len(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + ); + assert!( + recvd >= 0, + "recvfrom() failed: {}", + io::Error::last_os_error() + ); + + let recvd_slice = &buf[..(recvd as usize)]; + assert_eq!( + recvd_slice, + &msg[..], + "payload mismatch: sent {} bytes, got {} bytes", + msg.len(), + recvd + ); + + // Also exercise AF_UNIX stream socketpair quickly to ensure AF_UNIX in general works. + let mut sfds = [0i32; 2]; + let sr = libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, sfds.as_mut_ptr()); + assert_eq!( + sr, + 0, + "socketpair(AF_UNIX, SOCK_STREAM) failed: {}", + io::Error::last_os_error() + ); + let snt2 = libc::write(sfds[0], msg.as_ptr() as *const libc::c_void, msg.len()); + assert!( + snt2 >= 0, + "write(stream) failed: {}", + io::Error::last_os_error() + ); + let mut b2 = [0u8; 64]; + let rcv2 = libc::recv(sfds[1], b2.as_mut_ptr() as *mut libc::c_void, b2.len(), 0); + assert!( + rcv2 >= 0, + "recv(stream) failed: {}", + io::Error::last_os_error() + ); + + // Clean up + let _ = libc::close(sfds[0]); + let _ = libc::close(sfds[1]); + let _ = libc::close(fds[0]); + let _ = libc::close(fds[1]); + } +} + +#[tokio::test] +async fn allow_unix_socketpair_recvfrom() { + run_code_under_sandbox( + "allow_unix_socketpair_recvfrom", + &SandboxPolicy::ReadOnly, + || async { unix_sock_body() }, + ) + .await + .expect("should be able to reexec"); +} + +const IN_SANDBOX_ENV_VAR: &str = "IN_SANDBOX"; + +#[expect(clippy::expect_used)] +pub async fn run_code_under_sandbox( + test_selector: &str, + policy: &SandboxPolicy, + child_body: F, +) -> io::Result> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + Send + 'static, +{ + if std::env::var(IN_SANDBOX_ENV_VAR).is_err() { + let exe = std::env::current_exe()?; + let mut cmds = vec![exe.to_string_lossy().into_owned(), "--exact".into()]; + let mut stdio_policy = StdioPolicy::RedirectForShellTool; + // Allow for us to pass forward --nocapture / use the right stdio policy. + if std::env::args().any(|a| a == "--nocapture") { + cmds.push("--nocapture".into()); + stdio_policy = StdioPolicy::Inherit; + } + cmds.push(test_selector.into()); + + // Your existing launcher: + let mut child = spawn_command_under_sandbox( + cmds, + policy, + std::env::current_dir().expect("should be able to get current dir"), + stdio_policy, + HashMap::from([("IN_SANDBOX".into(), "1".into())]), + ) + .await?; + + let status = child.wait().await?; + Ok(Some(status)) + } else { + // Child branch: run the provided body. + child_body().await; + Ok(None) + } +} diff --git a/codex-rs/execpolicy/src/execv_checker.rs b/codex-rs/execpolicy/src/execv_checker.rs index 3c9084e8256..fcd80b2b5fd 100644 --- a/codex-rs/execpolicy/src/execv_checker.rs +++ b/codex-rs/execpolicy/src/execv_checker.rs @@ -140,7 +140,6 @@ fn is_executable_file(path: &str) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use tempfile::TempDir; use super::*; @@ -215,7 +214,12 @@ system_path=[{fake_cp:?}] // Only readable folders specified. assert_eq!( - checker.check(valid_exec.clone(), &cwd, &[root_path.clone()], &[]), + checker.check( + valid_exec.clone(), + &cwd, + std::slice::from_ref(&root_path), + &[] + ), Err(WriteablePathNotInWriteableFolders { file: dest_path.clone(), folders: vec![] @@ -227,8 +231,8 @@ system_path=[{fake_cp:?}] checker.check( valid_exec.clone(), &cwd, - &[root_path.clone()], - &[root_path.clone()] + std::slice::from_ref(&root_path), + std::slice::from_ref(&root_path) ), Ok(cp.clone()), ); @@ -247,8 +251,8 @@ system_path=[{fake_cp:?}] checker.check( valid_exec_call_folders_as_args, &cwd, - &[root_path.clone()], - &[root_path.clone()] + std::slice::from_ref(&root_path), + std::slice::from_ref(&root_path) ), Ok(cp.clone()), ); @@ -270,8 +274,8 @@ system_path=[{fake_cp:?}] checker.check( exec_with_parent_of_readable_folder, &cwd, - &[root_path.clone()], - &[dest_path.clone()] + std::slice::from_ref(&root_path), + std::slice::from_ref(&dest_path) ), Err(ReadablePathNotInReadableFolders { file: root_path.parent().unwrap().to_path_buf(), diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs index 7ce148b7c21..825d6164a56 100644 --- a/codex-rs/execpolicy/src/policy.rs +++ b/codex-rs/execpolicy/src/policy.rs @@ -56,16 +56,16 @@ impl Policy { } for arg in args { - if let Some(regex) = &self.forbidden_substrings_pattern { - if regex.is_match(arg) { - return Ok(MatchedExec::Forbidden { - cause: Forbidden::Arg { - arg: arg.clone(), - exec_call: exec_call.clone(), - }, - reason: format!("arg `{arg}` contains forbidden substring"), - }); - } + if let Some(regex) = &self.forbidden_substrings_pattern + && regex.is_match(arg) + { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Arg { + arg: arg.clone(), + exec_call: exec_call.clone(), + }, + reason: format!("arg `{arg}` contains forbidden substring"), + }); } } diff --git a/codex-rs/execpolicy/src/sed_command.rs b/codex-rs/execpolicy/src/sed_command.rs index 64494ddf00e..cc96aa98e7e 100644 --- a/codex-rs/execpolicy/src/sed_command.rs +++ b/codex-rs/execpolicy/src/sed_command.rs @@ -3,12 +3,12 @@ use crate::error::Result; pub fn parse_sed_command(sed_command: &str) -> Result<()> { // For now, we parse only commands like `122,202p`. - if let Some(stripped) = sed_command.strip_suffix("p") { - if let Some((first, rest)) = stripped.split_once(",") { - if first.parse::().is_ok() && rest.parse::().is_ok() { - return Ok(()); - } - } + if let Some(stripped) = sed_command.strip_suffix("p") + && let Some((first, rest)) = stripped.split_once(",") + && first.parse::().is_ok() + && rest.parse::().is_ok() + { + return Ok(()); } Err(Error::SedCommandNotProvablySafe { diff --git a/codex-rs/execpolicy/tests/bad.rs b/codex-rs/execpolicy/tests/bad.rs index 5c2999a0cbb..8b6e195fb05 100644 --- a/codex-rs/execpolicy/tests/bad.rs +++ b/codex-rs/execpolicy/tests/bad.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::NegativeExamplePassedCheck; use codex_execpolicy::get_default_policy; diff --git a/codex-rs/execpolicy/tests/cp.rs b/codex-rs/execpolicy/tests/cp.rs index 14fd24410f7..aa19f0b5d56 100644 --- a/codex-rs/execpolicy/tests/cp.rs +++ b/codex-rs/execpolicy/tests/cp.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgMatcher; @@ -12,6 +11,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/good.rs b/codex-rs/execpolicy/tests/good.rs index 645728afbdf..3b7313a335e 100644 --- a/codex-rs/execpolicy/tests/good.rs +++ b/codex-rs/execpolicy/tests/good.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::PositiveExampleFailedCheck; use codex_execpolicy::get_default_policy; diff --git a/codex-rs/execpolicy/tests/head.rs b/codex-rs/execpolicy/tests/head.rs index 8d7a165cca9..3c32ccfbfe4 100644 --- a/codex-rs/execpolicy/tests/head.rs +++ b/codex-rs/execpolicy/tests/head.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::ArgMatcher; use codex_execpolicy::ArgType; use codex_execpolicy::Error; @@ -13,6 +12,7 @@ use codex_execpolicy::get_default_policy; extern crate codex_execpolicy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/literal.rs b/codex-rs/execpolicy/tests/literal.rs index 629aeb9cfec..d849371e3bf 100644 --- a/codex-rs/execpolicy/tests/literal.rs +++ b/codex-rs/execpolicy/tests/literal.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::ArgType; use codex_execpolicy::Error; use codex_execpolicy::ExecCall; diff --git a/codex-rs/execpolicy/tests/ls.rs b/codex-rs/execpolicy/tests/ls.rs index 854ec3bf580..e52316c06e1 100644 --- a/codex-rs/execpolicy/tests/ls.rs +++ b/codex-rs/execpolicy/tests/ls.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgType; @@ -12,6 +11,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/pwd.rs b/codex-rs/execpolicy/tests/pwd.rs index 339908e9281..fdf5a4f1a50 100644 --- a/codex-rs/execpolicy/tests/pwd.rs +++ b/codex-rs/execpolicy/tests/pwd.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use std::vec; @@ -12,6 +11,7 @@ use codex_execpolicy::PositionalArg; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/sed.rs b/codex-rs/execpolicy/tests/sed.rs index 064e539305e..bf35bf6d460 100644 --- a/codex-rs/execpolicy/tests/sed.rs +++ b/codex-rs/execpolicy/tests/sed.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgType; @@ -13,6 +12,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/file-search/src/lib.rs b/codex-rs/file-search/src/lib.rs index e0eb87656be..bfcfe922ef0 100644 --- a/codex-rs/file-search/src/lib.rs +++ b/codex-rs/file-search/src/lib.rs @@ -228,11 +228,11 @@ pub fn run( for &Reverse((score, ref line)) in best_list.binary_heap.iter() { if global_heap.len() < limit.get() { global_heap.push(Reverse((score, line.clone()))); - } else if let Some(min_element) = global_heap.peek() { - if score > min_element.0.0 { - global_heap.pop(); - global_heap.push(Reverse((score, line.clone()))); - } + } else if let Some(min_element) = global_heap.peek() + && score > min_element.0.0 + { + global_heap.pop(); + global_heap.push(Reverse((score, line.clone()))); } } } @@ -320,11 +320,11 @@ impl BestMatchesList { if self.binary_heap.len() < self.max_count { self.binary_heap.push(Reverse((score, line.to_string()))); - } else if let Some(min_element) = self.binary_heap.peek() { - if score > min_element.0.0 { - self.binary_heap.pop(); - self.binary_heap.push(Reverse((score, line.to_string()))); - } + } else if let Some(min_element) = self.binary_heap.peek() + && score > min_element.0.0 + { + self.binary_heap.pop(); + self.binary_heap.push(Reverse((score, line.to_string()))); } } } diff --git a/codex-rs/justfile b/codex-rs/justfile index 3e1336be431..beaa682daa4 100644 --- a/codex-rs/justfile +++ b/codex-rs/justfile @@ -24,8 +24,8 @@ file-search *args: fmt: cargo fmt -- --config imports_granularity=Item -fix: - cargo clippy --fix --all-features --tests --allow-dirty +fix *args: + cargo clippy --fix --all-features --tests --allow-dirty "$@" install: rustup show active-toolchain diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index ea7052c4091..d769fae2c69 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -20,7 +20,7 @@ clap = { version = "4", features = ["derive"] } codex-common = { path = "../common", features = ["cli"] } codex-core = { path = "../core" } landlock = "0.4.1" -libc = "0.2.172" +libc = "0.2.175" seccompiler = "0.5.0" [target.'cfg(target_os = "linux")'.dev-dependencies] diff --git a/codex-rs/linux-sandbox/src/landlock.rs b/codex-rs/linux-sandbox/src/landlock.rs index e13e3c8b751..5bc96130dda 100644 --- a/codex-rs/linux-sandbox/src/landlock.rs +++ b/codex-rs/linux-sandbox/src/landlock.rs @@ -104,7 +104,9 @@ fn install_network_seccomp_filter_on_current_thread() -> std::result::Result<(), deny_syscall(libc::SYS_sendto); deny_syscall(libc::SYS_sendmsg); deny_syscall(libc::SYS_sendmmsg); - deny_syscall(libc::SYS_recvfrom); + // NOTE: allowing recvfrom allows some tools like: `cargo clippy` to run + // with their socketpair + child processes for sub-proc management + // deny_syscall(libc::SYS_recvfrom); deny_syscall(libc::SYS_recvmsg); deny_syscall(libc::SYS_recvmmsg); deny_syscall(libc::SYS_getsockopt); @@ -115,12 +117,12 @@ fn install_network_seccomp_filter_on_current_thread() -> std::result::Result<(), let unix_only_rule = SeccompRule::new(vec![SeccompCondition::new( 0, // first argument (domain) SeccompCmpArgLen::Dword, - SeccompCmpOp::Eq, + SeccompCmpOp::Ne, libc::AF_UNIX as u64, )?])?; - rules.insert(libc::SYS_socket, vec![unix_only_rule]); - rules.insert(libc::SYS_socketpair, vec![]); // always deny (Unix can use socketpair but fine, keep open?) + rules.insert(libc::SYS_socket, vec![unix_only_rule.clone()]); + rules.insert(libc::SYS_socketpair, vec![unix_only_rule]); // always deny (Unix can use socketpair but fine, keep open?) let filter = SeccompFilter::new( rules, diff --git a/codex-rs/linux-sandbox/tests/landlock.rs b/codex-rs/linux-sandbox/tests/landlock.rs index c7081dbca2f..8faa02d1585 100644 --- a/codex-rs/linux-sandbox/tests/landlock.rs +++ b/codex-rs/linux-sandbox/tests/landlock.rs @@ -1,6 +1,4 @@ #![cfg(target_os = "linux")] -#![expect(clippy::unwrap_used, clippy::expect_used)] - use codex_core::config_types::ShellEnvironmentPolicy; use codex_core::error::CodexErr; use codex_core::error::SandboxErr; @@ -11,9 +9,7 @@ use codex_core::exec_env::create_env; use codex_core::protocol::SandboxPolicy; use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; use tempfile::NamedTempFile; -use tokio::sync::Notify; // At least on GitHub CI, the arm64 tests appear to need longer timeouts. @@ -37,7 +33,7 @@ fn create_env_from_core_vars() -> HashMap { create_env(&policy) } -#[allow(clippy::print_stdout)] +#[expect(clippy::print_stdout, clippy::expect_used, clippy::unwrap_used)] async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { let params = ExecParams { command: cmd.iter().map(|elm| elm.to_string()).collect(), @@ -59,11 +55,9 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { }; let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox"); let codex_linux_sandbox_exe = Some(PathBuf::from(sandbox_program)); - let ctrl_c = Arc::new(Notify::new()); let res = process_exec_tool_call( params, SandboxType::LinuxSeccomp, - ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, None, @@ -136,6 +130,7 @@ async fn test_timeout() { /// does NOT succeed (i.e. returns a non‑zero exit code) **unless** the binary /// is missing in which case we silently treat it as an accepted skip so the /// suite remains green on leaner CI images. +#[expect(clippy::expect_used)] async fn assert_network_blocked(cmd: &[&str]) { let cwd = std::env::current_dir().expect("cwd should exist"); let params = ExecParams { @@ -150,13 +145,11 @@ async fn assert_network_blocked(cmd: &[&str]) { }; let sandbox_policy = SandboxPolicy::new_read_only_policy(); - let ctrl_c = Arc::new(Notify::new()); let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox"); let codex_linux_sandbox_exe: Option = Some(PathBuf::from(sandbox_program)); let result = process_exec_tool_call( params, SandboxType::LinuxSeccomp, - ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, None, diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index 85c11505ec3..bf04a8e3ff2 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -9,11 +9,15 @@ workspace = true [dependencies] base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } -reqwest = { version = "0.12", features = ["json"] } +codex-protocol = { path = "../protocol" } +rand = "0.8" +reqwest = { version = "0.12", features = ["json", "blocking"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" tempfile = "3" thiserror = "2.0.12" +tiny_http = "0.12" tokio = { version = "1", features = [ "io-std", "macros", @@ -21,6 +25,9 @@ tokio = { version = "1", features = [ "rt-multi-thread", "signal", ] } +url = "2" +urlencoding = "2.1" +webbrowser = "1.0" [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/codex-rs/login/src/assets/success.html b/codex-rs/login/src/assets/success.html new file mode 100644 index 00000000000..eb2a0ee719c --- /dev/null +++ b/codex-rs/login/src/assets/success.html @@ -0,0 +1,198 @@ + + + + + Sign into Codex CLI + + + + +

+
+
+ +
Signed in to Codex CLI
+
+ + +
+
+ + + + \ No newline at end of file diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index a1dad79e9db..1f118823024 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -1,5 +1,4 @@ use chrono::DateTime; - use chrono::Utc; use serde::Deserialize; use serde::Serialize; @@ -9,34 +8,28 @@ use std::fs::OpenOptions; use std::fs::remove_file; use std::io::Read; use std::io::Write; -use std::io::{self}; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::path::PathBuf; -use std::process::Child; -use std::process::Stdio; use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; -use tempfile::NamedTempFile; -use tokio::process::Command; +pub use crate::server::LoginServer; +pub use crate::server::ServerOptions; +pub use crate::server::ShutdownHandle; +pub use crate::server::run_login_server; pub use crate::token_data::TokenData; use crate::token_data::parse_id_token; +mod pkce; +mod server; mod token_data; -const SOURCE_FOR_PYTHON_SERVER: &str = include_str!("./login_with_chatgpt.py"); - -const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; +pub const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; pub const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; - -#[derive(Clone, Debug, PartialEq, Copy)] -pub enum AuthMode { - ApiKey, - ChatGPT, -} +pub use codex_protocol::mcp_protocol::AuthMode; #[derive(Debug, Clone)] pub struct CodexAuth { @@ -63,10 +56,46 @@ impl CodexAuth { } } + pub async fn refresh_token(&self) -> Result { + let token_data = self + .get_current_token_data() + .ok_or(std::io::Error::other("Token data is not available."))?; + let token = token_data.refresh_token; + + let refresh_response = try_refresh_token(token) + .await + .map_err(std::io::Error::other)?; + + let updated = update_tokens( + &self.auth_file, + refresh_response.id_token, + refresh_response.access_token, + refresh_response.refresh_token, + ) + .await?; + + if let Ok(mut auth_lock) = self.auth_dot_json.lock() { + *auth_lock = Some(updated.clone()); + } + + let access = match updated.tokens { + Some(t) => t.access_token, + None => { + return Err(std::io::Error::other( + "Token data is not available after refresh.", + )); + } + }; + Ok(access) + } + /// Loads the available auth information from the auth.json or /// OPENAI_API_KEY environment variable. - pub fn from_codex_home(codex_home: &Path) -> std::io::Result> { - load_auth(codex_home, true) + pub fn from_codex_home( + codex_home: &Path, + preferred_auth_method: AuthMode, + ) -> std::io::Result> { + load_auth(codex_home, true, preferred_auth_method) } pub async fn get_token_data(&self) -> Result { @@ -167,7 +196,11 @@ impl CodexAuth { } } -fn load_auth(codex_home: &Path, include_env_var: bool) -> std::io::Result> { +fn load_auth( + codex_home: &Path, + include_env_var: bool, + preferred_auth_method: AuthMode, +) -> std::io::Result> { // First, check to see if there is a valid auth.json file. If not, we fall // back to AuthMode::ApiKey using the OPENAI_API_KEY environment variable // (if it is set). @@ -203,7 +236,7 @@ fn load_auth(codex_home: &Path, include_env_var: bool) -> std::io::Result