From e65f5eea2961409984d992e240f090032ebd6501 Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Tue, 10 Mar 2026 08:41:41 -0700 Subject: [PATCH 01/41] Refactor config access around an Ecto schema #### Context The config layer had grown around NimbleOptions and a large getter surface. This branch moves parsing to an Ecto schema and simplifies callers to read typed nested settings directly. #### TL;DR *Replace the bespoke config access layer with an Ecto-backed schema and typed settings access.* #### Summary - Replace the old config extraction path with an Ecto embedded schema and defaults - Collapse `Config` down to `settings/0`, `settings!/0`, and a small runtime helper surface - Update callers and tests to read nested config structs directly instead of one-off getter wrappers - Keep the repo green under format, lint, coverage, and dialyzer by adding the missing schema specs and tests #### Alternatives - Keep NimbleOptions and shrink the wrapper layer incrementally, but that would preserve the custom parser shape - Keep the strict refactor without extra schema tests, but that left `make -C elixir all` red on coverage and dialyzer #### Test Plan - [x] `make -C elixir all` - [x] `cd elixir && mix test test/symphony_elixir/workspace_and_config_test.exs --warnings-as-errors` --- elixir/lib/symphony_elixir/agent_runner.ex | 4 +- .../lib/symphony_elixir/codex/app_server.ex | 15 +- elixir/lib/symphony_elixir/config.ex | 909 ++---------------- elixir/lib/symphony_elixir/config/schema.ex | 493 ++++++++++ elixir/lib/symphony_elixir/http_server.ex | 2 +- elixir/lib/symphony_elixir/linear/client.ex | 18 +- elixir/lib/symphony_elixir/orchestrator.ex | 40 +- .../lib/symphony_elixir/status_dashboard.ex | 21 +- elixir/lib/symphony_elixir/tracker.ex | 2 +- elixir/lib/symphony_elixir/workspace.ex | 22 +- elixir/lib/symphony_elixir_web/presenter.ex | 2 +- elixir/mix.exs | 2 +- elixir/mix.lock | 1 + elixir/test/symphony_elixir/core_test.exs | 49 +- .../test/symphony_elixir/extensions_test.exs | 4 +- .../workspace_and_config_test.exs | 234 +++-- 16 files changed, 823 insertions(+), 995 deletions(-) create mode 100644 elixir/lib/symphony_elixir/config/schema.ex diff --git a/elixir/lib/symphony_elixir/agent_runner.ex b/elixir/lib/symphony_elixir/agent_runner.ex index 7292a4bc27..146350602c 100644 --- a/elixir/lib/symphony_elixir/agent_runner.ex +++ b/elixir/lib/symphony_elixir/agent_runner.ex @@ -47,7 +47,7 @@ defmodule SymphonyElixir.AgentRunner do defp send_codex_update(_recipient, _issue, _message), do: :ok defp run_codex_turns(workspace, issue, codex_update_recipient, opts) do - max_turns = Keyword.get(opts, :max_turns, Config.agent_max_turns()) + max_turns = Keyword.get(opts, :max_turns, Config.settings!().agent.max_turns) issue_state_fetcher = Keyword.get(opts, :issue_state_fetcher, &Tracker.fetch_issue_states_by_ids/1) with {:ok, session} <- AppServer.start_session(workspace) do @@ -136,7 +136,7 @@ defmodule SymphonyElixir.AgentRunner do defp active_issue_state?(state_name) when is_binary(state_name) do normalized_state = normalize_issue_state(state_name) - Config.linear_active_states() + Config.settings!().tracker.active_states |> Enum.any?(fn active_state -> normalize_issue_state(active_state) == normalized_state end) end diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index e824c63b8d..1f9252e2f1 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -143,7 +143,7 @@ defmodule SymphonyElixir.Codex.AppServer do defp validate_workspace_cwd(workspace) when is_binary(workspace) do workspace_path = Path.expand(workspace) - workspace_root = Path.expand(Config.workspace_root()) + workspace_root = Path.expand(Config.settings!().workspace.root) root_prefix = workspace_root <> "/" @@ -172,7 +172,7 @@ defmodule SymphonyElixir.Codex.AppServer do :binary, :exit_status, :stderr_to_stdout, - args: [~c"-lc", String.to_charlist(Config.codex_command())], + args: [~c"-lc", String.to_charlist(Config.settings!().codex.command)], cd: String.to_charlist(workspace), line: @port_line_bytes ] @@ -277,7 +277,14 @@ defmodule SymphonyElixir.Codex.AppServer do end defp await_turn_completion(port, on_message, tool_executor, auto_approve_requests) do - receive_loop(port, on_message, Config.codex_turn_timeout_ms(), "", tool_executor, auto_approve_requests) + receive_loop( + port, + on_message, + Config.settings!().codex.turn_timeout_ms, + "", + tool_executor, + auto_approve_requests + ) end defp receive_loop(port, on_message, timeout_ms, pending_line, tool_executor, auto_approve_requests) do @@ -820,7 +827,7 @@ defmodule SymphonyElixir.Codex.AppServer do end defp await_response(port, request_id) do - with_timeout_response(port, request_id, Config.codex_read_timeout_ms(), "") + with_timeout_response(port, request_id, Config.settings!().codex.read_timeout_ms, "") end defp with_timeout_response(port, request_id, timeout_ms, pending_line) do diff --git a/elixir/lib/symphony_elixir/config.ex b/elixir/lib/symphony_elixir/config.ex index 3a9f0d997a..84623ca23c 100644 --- a/elixir/lib/symphony_elixir/config.ex +++ b/elixir/lib/symphony_elixir/config.ex @@ -3,12 +3,9 @@ defmodule SymphonyElixir.Config do Runtime configuration loaded from `WORKFLOW.md`. """ - alias NimbleOptions + alias SymphonyElixir.Config.Schema alias SymphonyElixir.Workflow - @default_active_states ["Todo", "In Progress"] - @default_terminal_states ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"] - @default_linear_endpoint "https://api.linear.app/graphql" @default_prompt_template """ You are working on a Linear issue. @@ -22,306 +19,57 @@ defmodule SymphonyElixir.Config do No description provided. {% endif %} """ - @default_poll_interval_ms 30_000 - @default_workspace_root Path.join(System.tmp_dir!(), "symphony_workspaces") - @default_hook_timeout_ms 60_000 - @default_max_concurrent_agents 10 - @default_agent_max_turns 20 - @default_max_retry_backoff_ms 300_000 - @default_codex_command "codex app-server" - @default_codex_turn_timeout_ms 3_600_000 - @default_codex_read_timeout_ms 5_000 - @default_codex_stall_timeout_ms 300_000 - @default_codex_approval_policy %{ - "reject" => %{ - "sandbox_approval" => true, - "rules" => true, - "mcp_elicitations" => true - } - } - @default_codex_thread_sandbox "workspace-write" - @default_observability_enabled true - @default_observability_refresh_ms 1_000 - @default_observability_render_interval_ms 16 - @default_server_host "127.0.0.1" - @workflow_options_schema NimbleOptions.new!( - tracker: [ - type: :map, - default: %{}, - keys: [ - kind: [type: {:or, [:string, nil]}, default: nil], - endpoint: [type: :string, default: @default_linear_endpoint], - api_key: [type: {:or, [:string, nil]}, default: nil], - project_slug: [type: {:or, [:string, nil]}, default: nil], - assignee: [type: {:or, [:string, nil]}, default: nil], - active_states: [ - type: {:list, :string}, - default: @default_active_states - ], - terminal_states: [ - type: {:list, :string}, - default: @default_terminal_states - ] - ] - ], - polling: [ - type: :map, - default: %{}, - keys: [ - interval_ms: [type: :integer, default: @default_poll_interval_ms] - ] - ], - workspace: [ - type: :map, - default: %{}, - keys: [ - root: [type: {:or, [:string, nil]}, default: @default_workspace_root] - ] - ], - agent: [ - type: :map, - default: %{}, - keys: [ - max_concurrent_agents: [ - type: :integer, - default: @default_max_concurrent_agents - ], - max_turns: [ - type: :pos_integer, - default: @default_agent_max_turns - ], - max_retry_backoff_ms: [ - type: :pos_integer, - default: @default_max_retry_backoff_ms - ], - max_concurrent_agents_by_state: [ - type: {:map, :string, :pos_integer}, - default: %{} - ] - ] - ], - codex: [ - type: :map, - default: %{}, - keys: [ - command: [type: :string, default: @default_codex_command], - turn_timeout_ms: [ - type: :integer, - default: @default_codex_turn_timeout_ms - ], - read_timeout_ms: [ - type: :integer, - default: @default_codex_read_timeout_ms - ], - stall_timeout_ms: [ - type: :integer, - default: @default_codex_stall_timeout_ms - ] - ] - ], - hooks: [ - type: :map, - default: %{}, - keys: [ - after_create: [type: {:or, [:string, nil]}, default: nil], - before_run: [type: {:or, [:string, nil]}, default: nil], - after_run: [type: {:or, [:string, nil]}, default: nil], - before_remove: [type: {:or, [:string, nil]}, default: nil], - timeout_ms: [type: :pos_integer, default: @default_hook_timeout_ms] - ] - ], - observability: [ - type: :map, - default: %{}, - keys: [ - dashboard_enabled: [ - type: :boolean, - default: @default_observability_enabled - ], - refresh_ms: [ - type: :integer, - default: @default_observability_refresh_ms - ], - render_interval_ms: [ - type: :integer, - default: @default_observability_render_interval_ms - ] - ] - ], - server: [ - type: :map, - default: %{}, - keys: [ - port: [type: {:or, [:non_neg_integer, nil]}, default: nil], - host: [type: :string, default: @default_server_host] - ] - ] - ) - @type workflow_payload :: Workflow.loaded_workflow() - @type tracker_kind :: String.t() | nil @type codex_runtime_settings :: %{ approval_policy: String.t() | map(), thread_sandbox: String.t(), turn_sandbox_policy: map() } - @type workspace_hooks :: %{ - after_create: String.t() | nil, - before_run: String.t() | nil, - after_run: String.t() | nil, - before_remove: String.t() | nil, - timeout_ms: pos_integer() - } - - @spec current_workflow() :: {:ok, workflow_payload()} | {:error, term()} - def current_workflow do - Workflow.current() - end - - @spec tracker_kind() :: tracker_kind() - def tracker_kind do - get_in(validated_workflow_options(), [:tracker, :kind]) - end - - @spec linear_endpoint() :: String.t() - def linear_endpoint do - get_in(validated_workflow_options(), [:tracker, :endpoint]) - end - - @spec linear_api_token() :: String.t() | nil - def linear_api_token do - validated_workflow_options() - |> get_in([:tracker, :api_key]) - |> resolve_env_value(System.get_env("LINEAR_API_KEY")) - |> normalize_secret_value() - end - - @spec linear_project_slug() :: String.t() | nil - def linear_project_slug do - get_in(validated_workflow_options(), [:tracker, :project_slug]) - end - - @spec linear_assignee() :: String.t() | nil - def linear_assignee do - validated_workflow_options() - |> get_in([:tracker, :assignee]) - |> resolve_env_value(System.get_env("LINEAR_ASSIGNEE")) - |> normalize_secret_value() - end - - @spec linear_active_states() :: [String.t()] - def linear_active_states do - get_in(validated_workflow_options(), [:tracker, :active_states]) - end - @spec linear_terminal_states() :: [String.t()] - def linear_terminal_states do - get_in(validated_workflow_options(), [:tracker, :terminal_states]) - end - - @spec poll_interval_ms() :: pos_integer() - def poll_interval_ms do - get_in(validated_workflow_options(), [:polling, :interval_ms]) - end - - @spec workspace_root() :: Path.t() - def workspace_root do - validated_workflow_options() - |> get_in([:workspace, :root]) - |> resolve_path_value(@default_workspace_root) - end - - @spec workspace_hooks() :: workspace_hooks() - def workspace_hooks do - hooks = get_in(validated_workflow_options(), [:hooks]) - - %{ - after_create: Map.get(hooks, :after_create), - before_run: Map.get(hooks, :before_run), - after_run: Map.get(hooks, :after_run), - before_remove: Map.get(hooks, :before_remove), - timeout_ms: Map.get(hooks, :timeout_ms) - } - end + @spec settings() :: {:ok, Schema.t()} | {:error, term()} + def settings do + case Workflow.current() do + {:ok, %{config: config}} when is_map(config) -> + Schema.parse(config) - @spec hook_timeout_ms() :: pos_integer() - def hook_timeout_ms do - get_in(validated_workflow_options(), [:hooks, :timeout_ms]) + {:error, reason} -> + {:error, reason} + end end - @spec max_concurrent_agents() :: pos_integer() - def max_concurrent_agents do - get_in(validated_workflow_options(), [:agent, :max_concurrent_agents]) - end + @spec settings!() :: Schema.t() + def settings! do + case settings() do + {:ok, settings} -> + settings - @spec max_retry_backoff_ms() :: pos_integer() - def max_retry_backoff_ms do - get_in(validated_workflow_options(), [:agent, :max_retry_backoff_ms]) - end - - @spec agent_max_turns() :: pos_integer() - def agent_max_turns do - get_in(validated_workflow_options(), [:agent, :max_turns]) + {:error, reason} -> + raise ArgumentError, message: format_config_error(reason) + end end @spec max_concurrent_agents_for_state(term()) :: pos_integer() def max_concurrent_agents_for_state(state_name) when is_binary(state_name) do - state_limits = get_in(validated_workflow_options(), [:agent, :max_concurrent_agents_by_state]) - global_limit = max_concurrent_agents() - Map.get(state_limits, normalize_issue_state(state_name), global_limit) - end + config = settings!() - def max_concurrent_agents_for_state(_state_name), do: max_concurrent_agents() - - @spec codex_command() :: String.t() - def codex_command do - get_in(validated_workflow_options(), [:codex, :command]) - end - - @spec codex_turn_timeout_ms() :: pos_integer() - def codex_turn_timeout_ms do - get_in(validated_workflow_options(), [:codex, :turn_timeout_ms]) + Map.get( + config.agent.max_concurrent_agents_by_state, + Schema.normalize_issue_state(state_name), + config.agent.max_concurrent_agents + ) end - @spec codex_approval_policy() :: String.t() | map() - def codex_approval_policy do - case resolve_codex_approval_policy() do - {:ok, approval_policy} -> approval_policy - {:error, _reason} -> @default_codex_approval_policy - end - end - - @spec codex_thread_sandbox() :: String.t() - def codex_thread_sandbox do - case resolve_codex_thread_sandbox() do - {:ok, thread_sandbox} -> thread_sandbox - {:error, _reason} -> @default_codex_thread_sandbox - end - end + def max_concurrent_agents_for_state(_state_name), do: settings!().agent.max_concurrent_agents @spec codex_turn_sandbox_policy(Path.t() | nil) :: map() def codex_turn_sandbox_policy(workspace \\ nil) do - case resolve_codex_turn_sandbox_policy(workspace) do - {:ok, turn_sandbox_policy} -> turn_sandbox_policy - {:error, _reason} -> default_codex_turn_sandbox_policy(workspace) - end - end - - @spec codex_read_timeout_ms() :: pos_integer() - def codex_read_timeout_ms do - get_in(validated_workflow_options(), [:codex, :read_timeout_ms]) - end - - @spec codex_stall_timeout_ms() :: non_neg_integer() - def codex_stall_timeout_ms do - validated_workflow_options() - |> get_in([:codex, :stall_timeout_ms]) - |> max(0) + settings!() + |> Schema.resolve_turn_sandbox_policy(workspace) end @spec workflow_prompt() :: String.t() def workflow_prompt do - case current_workflow() do + case Workflow.current() do {:ok, %{prompt_template: prompt}} -> if String.trim(prompt) == "", do: @default_prompt_template, else: prompt @@ -330,609 +78,68 @@ defmodule SymphonyElixir.Config do end end - @spec observability_enabled?() :: boolean() - def observability_enabled? do - get_in(validated_workflow_options(), [:observability, :dashboard_enabled]) - end - - @spec observability_refresh_ms() :: pos_integer() - def observability_refresh_ms do - get_in(validated_workflow_options(), [:observability, :refresh_ms]) - end - - @spec observability_render_interval_ms() :: pos_integer() - def observability_render_interval_ms do - get_in(validated_workflow_options(), [:observability, :render_interval_ms]) - end - @spec server_port() :: non_neg_integer() | nil def server_port do case Application.get_env(:symphony_elixir, :server_port_override) do - port when is_integer(port) and port >= 0 -> - port - - _ -> - get_in(validated_workflow_options(), [:server, :port]) + port when is_integer(port) and port >= 0 -> port + _ -> settings!().server.port end end - @spec server_host() :: String.t() - def server_host do - get_in(validated_workflow_options(), [:server, :host]) - end - @spec validate!() :: :ok | {:error, term()} def validate! do - with {:ok, _workflow} <- current_workflow(), - :ok <- require_tracker_kind(), - :ok <- require_linear_token(), - :ok <- require_linear_project(), - :ok <- require_valid_codex_runtime_settings() do - require_codex_command() + with {:ok, settings} <- settings() do + validate_semantics(settings) end end @spec codex_runtime_settings(Path.t() | nil) :: {:ok, codex_runtime_settings()} | {:error, term()} def codex_runtime_settings(workspace \\ nil) do - with {:ok, approval_policy} <- resolve_codex_approval_policy(), - {:ok, thread_sandbox} <- resolve_codex_thread_sandbox(), - {:ok, turn_sandbox_policy} <- resolve_codex_turn_sandbox_policy(workspace) do + with {:ok, settings} <- settings() do {:ok, %{ - approval_policy: approval_policy, - thread_sandbox: thread_sandbox, - turn_sandbox_policy: turn_sandbox_policy + approval_policy: settings.codex.approval_policy, + thread_sandbox: settings.codex.thread_sandbox, + turn_sandbox_policy: Schema.resolve_turn_sandbox_policy(settings, workspace) }} end end - defp require_tracker_kind do - case tracker_kind() do - "linear" -> :ok - "memory" -> :ok - nil -> {:error, :missing_tracker_kind} - other -> {:error, {:unsupported_tracker_kind, other}} - end - end - - defp require_linear_token do - case tracker_kind() do - "linear" -> - if is_binary(linear_api_token()) do - :ok - else - {:error, :missing_linear_api_token} - end - - _ -> - :ok - end - end - - defp require_linear_project do - case tracker_kind() do - "linear" -> - if is_binary(linear_project_slug()) do - :ok - else - {:error, :missing_linear_project_slug} - end - - _ -> - :ok - end - end - - defp require_codex_command do - if byte_size(String.trim(codex_command())) > 0 do - :ok - else - {:error, :missing_codex_command} - end - end - - defp require_valid_codex_runtime_settings do - case codex_runtime_settings() do - {:ok, _settings} -> :ok - {:error, reason} -> {:error, reason} - end - end - - defp validated_workflow_options do - workflow_config() - |> extract_workflow_options() - |> NimbleOptions.validate!(@workflow_options_schema) - end - - defp extract_workflow_options(config) do - %{ - tracker: extract_tracker_options(section_map(config, "tracker")), - polling: extract_polling_options(section_map(config, "polling")), - workspace: extract_workspace_options(section_map(config, "workspace")), - agent: extract_agent_options(section_map(config, "agent")), - codex: extract_codex_options(section_map(config, "codex")), - hooks: extract_hooks_options(section_map(config, "hooks")), - observability: extract_observability_options(section_map(config, "observability")), - server: extract_server_options(section_map(config, "server")) - } - end - - defp extract_tracker_options(section) do - %{} - |> put_if_present(:kind, normalize_tracker_kind(scalar_string_value(Map.get(section, "kind")))) - |> put_if_present(:endpoint, scalar_string_value(Map.get(section, "endpoint"))) - |> put_if_present(:api_key, binary_value(Map.get(section, "api_key"), allow_empty: true)) - |> put_if_present(:project_slug, scalar_string_value(Map.get(section, "project_slug"))) - |> put_if_present(:active_states, csv_value(Map.get(section, "active_states"))) - |> put_if_present(:terminal_states, csv_value(Map.get(section, "terminal_states"))) - end - - defp extract_polling_options(section) do - %{} - |> put_if_present(:interval_ms, integer_value(Map.get(section, "interval_ms"))) - end - - defp extract_workspace_options(section) do - %{} - |> put_if_present(:root, binary_value(Map.get(section, "root"))) - end - - defp extract_agent_options(section) do - %{} - |> put_if_present(:max_concurrent_agents, integer_value(Map.get(section, "max_concurrent_agents"))) - |> put_if_present(:max_turns, positive_integer_value(Map.get(section, "max_turns"))) - |> put_if_present(:max_retry_backoff_ms, positive_integer_value(Map.get(section, "max_retry_backoff_ms"))) - |> put_if_present( - :max_concurrent_agents_by_state, - state_limits_value(Map.get(section, "max_concurrent_agents_by_state")) - ) - end - - defp extract_codex_options(section) do - %{} - |> put_if_present(:command, command_value(Map.get(section, "command"))) - |> put_if_present(:turn_timeout_ms, integer_value(Map.get(section, "turn_timeout_ms"))) - |> put_if_present(:read_timeout_ms, integer_value(Map.get(section, "read_timeout_ms"))) - |> put_if_present(:stall_timeout_ms, integer_value(Map.get(section, "stall_timeout_ms"))) - end - - defp extract_hooks_options(section) do - %{} - |> put_if_present(:after_create, hook_command_value(Map.get(section, "after_create"))) - |> put_if_present(:before_run, hook_command_value(Map.get(section, "before_run"))) - |> put_if_present(:after_run, hook_command_value(Map.get(section, "after_run"))) - |> put_if_present(:before_remove, hook_command_value(Map.get(section, "before_remove"))) - |> put_if_present(:timeout_ms, positive_integer_value(Map.get(section, "timeout_ms"))) - end - - defp extract_observability_options(section) do - %{} - |> put_if_present(:dashboard_enabled, boolean_value(Map.get(section, "dashboard_enabled"))) - |> put_if_present(:refresh_ms, integer_value(Map.get(section, "refresh_ms"))) - |> put_if_present(:render_interval_ms, integer_value(Map.get(section, "render_interval_ms"))) - end - - defp extract_server_options(section) do - %{} - |> put_if_present(:port, non_negative_integer_value(Map.get(section, "port"))) - |> put_if_present(:host, scalar_string_value(Map.get(section, "host"))) - end - - defp section_map(config, key) do - case Map.get(config, key) do - section when is_map(section) -> section - _ -> %{} - end - end - - defp put_if_present(map, _key, :omit), do: map - defp put_if_present(map, key, value), do: Map.put(map, key, value) - - defp scalar_string_value(nil), do: :omit - defp scalar_string_value(value) when is_binary(value), do: String.trim(value) - defp scalar_string_value(value) when is_boolean(value), do: to_string(value) - defp scalar_string_value(value) when is_integer(value), do: to_string(value) - defp scalar_string_value(value) when is_float(value), do: to_string(value) - defp scalar_string_value(value) when is_atom(value), do: Atom.to_string(value) - defp scalar_string_value(_value), do: :omit - - defp binary_value(value, opts \\ []) - - defp binary_value(value, opts) when is_binary(value) do - allow_empty = Keyword.get(opts, :allow_empty, false) - - if value == "" and not allow_empty do - :omit - else - value - end - end - - defp binary_value(_value, _opts), do: :omit - - defp command_value(value) when is_binary(value) do - case String.trim(value) do - "" -> :omit - trimmed -> trimmed - end - end - - defp command_value(_value), do: :omit - - defp hook_command_value(value) when is_binary(value) do - case String.trim(value) do - "" -> :omit - _ -> String.trim_trailing(value) - end - end - - defp hook_command_value(_value), do: :omit - - defp csv_value(values) when is_list(values) do - values - |> Enum.reduce([], fn value, acc -> maybe_append_csv_value(acc, value) end) - |> Enum.reverse() - |> case do - [] -> :omit - normalized_values -> normalized_values - end - end - - defp csv_value(value) when is_binary(value) do - value - |> String.split(",", trim: true) - |> Enum.map(&String.trim/1) - |> Enum.reject(&(&1 == "")) - |> case do - [] -> :omit - normalized_values -> normalized_values - end - end - - defp csv_value(_value), do: :omit - - defp maybe_append_csv_value(acc, value) do - case scalar_string_value(value) do - :omit -> - acc - - normalized -> - append_csv_value_if_present(acc, normalized) - end - end - - defp append_csv_value_if_present(acc, value) do - trimmed = String.trim(value) - - if trimmed == "" do - acc - else - [trimmed | acc] - end - end - - defp integer_value(value) do - case parse_integer(value) do - {:ok, parsed} -> parsed - :error -> :omit - end - end - - defp positive_integer_value(value) do - case parse_positive_integer(value) do - {:ok, parsed} -> parsed - :error -> :omit - end - end - - defp non_negative_integer_value(value) do - case parse_non_negative_integer(value) do - {:ok, parsed} -> parsed - :error -> :omit - end - end - - defp boolean_value(value) when is_boolean(value), do: value - - defp boolean_value(value) when is_binary(value) do - case String.downcase(String.trim(value)) do - "true" -> true - "false" -> false - _ -> :omit - end - end - - defp boolean_value(_value), do: :omit - - defp state_limits_value(value) when is_map(value) do - value - |> Enum.reduce(%{}, fn {state_name, limit}, acc -> - case parse_positive_integer(limit) do - {:ok, parsed} -> - Map.put(acc, normalize_issue_state(to_string(state_name)), parsed) - - :error -> - acc - end - end) - end - - defp state_limits_value(_value), do: :omit - - defp parse_integer(value) when is_integer(value), do: {:ok, value} - - defp parse_integer(value) when is_binary(value) do - case Integer.parse(String.trim(value)) do - {parsed, _} -> {:ok, parsed} - :error -> :error - end - end - - defp parse_integer(_value), do: :error - - defp parse_positive_integer(value) do - case parse_integer(value) do - {:ok, parsed} when parsed > 0 -> {:ok, parsed} - _ -> :error - end - end - - defp parse_non_negative_integer(value) do - case parse_integer(value) do - {:ok, parsed} when parsed >= 0 -> {:ok, parsed} - _ -> :error - end - end - - defp fetch_value(paths, default) do - config = workflow_config() - - case resolve_config_value(config, paths) do - :missing -> default - value -> value - end - end - - defp resolve_codex_approval_policy do - case fetch_value([["codex", "approval_policy"]], :missing) do - :missing -> - {:ok, @default_codex_approval_policy} - - nil -> - {:ok, @default_codex_approval_policy} - - value when is_binary(value) -> - approval_policy = String.trim(value) - - if approval_policy == "" do - {:error, {:invalid_codex_approval_policy, value}} - else - {:ok, approval_policy} - end - - value when is_map(value) -> - {:ok, value} - - value -> - {:error, {:invalid_codex_approval_policy, value}} - end - end - - defp resolve_codex_thread_sandbox do - case fetch_value([["codex", "thread_sandbox"]], :missing) do - :missing -> - {:ok, @default_codex_thread_sandbox} - - nil -> - {:ok, @default_codex_thread_sandbox} - - value when is_binary(value) -> - thread_sandbox = String.trim(value) - - if thread_sandbox == "" do - {:error, {:invalid_codex_thread_sandbox, value}} - else - {:ok, thread_sandbox} - end - - value -> - {:error, {:invalid_codex_thread_sandbox, value}} - end - end - - defp resolve_codex_turn_sandbox_policy(workspace) do - case fetch_value([["codex", "turn_sandbox_policy"]], :missing) do - :missing -> - {:ok, default_codex_turn_sandbox_policy(workspace)} - - nil -> - {:ok, default_codex_turn_sandbox_policy(workspace)} - - value when is_map(value) -> - {:ok, value} - - value -> - {:error, {:invalid_codex_turn_sandbox_policy, {:unsupported_value, value}}} - end - end - - defp default_codex_turn_sandbox_policy(workspace) do - writable_root = - if is_binary(workspace) and String.trim(workspace) != "" do - Path.expand(workspace) - else - Path.expand(workspace_root()) - end - - %{ - "type" => "workspaceWrite", - "writableRoots" => [writable_root], - "readOnlyAccess" => %{"type" => "fullAccess"}, - "networkAccess" => false, - "excludeTmpdirEnvVar" => false, - "excludeSlashTmp" => false - } - end - - defp normalize_issue_state(state_name) when is_binary(state_name) do - state_name - |> String.trim() - |> String.downcase() - end - - defp normalize_tracker_kind(kind) when is_binary(kind) do - kind - |> String.trim() - |> String.downcase() - |> case do - "" -> nil - normalized -> normalized - end - end - - defp normalize_tracker_kind(_kind), do: nil - - defp workflow_config do - case current_workflow() do - {:ok, %{config: config}} when is_map(config) -> - normalize_keys(config) - - _ -> - %{} - end - end - - defp resolve_config_value(%{} = config, paths) do - Enum.reduce_while(paths, :missing, fn path, _acc -> - case get_in_path(config, path) do - :missing -> {:cont, :missing} - value -> {:halt, value} - end - end) - end - - defp get_in_path(config, path) when is_list(path) and is_map(config) do - get_in_path(config, path, 0) - end - - defp get_in_path(_, _), do: :missing - - defp get_in_path(config, [], _depth), do: config - - defp get_in_path(%{} = current, [segment | rest], _depth) do - case Map.fetch(current, normalize_key(segment)) do - {:ok, value} -> get_in_path(value, rest, 0) - :error -> :missing - end - end - - defp get_in_path(_, _, _depth), do: :missing - - defp normalize_keys(value) when is_map(value) do - Enum.reduce(value, %{}, fn {key, raw_value}, normalized -> - Map.put(normalized, normalize_key(key), normalize_keys(raw_value)) - end) - end - - defp normalize_keys(value) when is_list(value), do: Enum.map(value, &normalize_keys/1) - defp normalize_keys(value), do: value - - defp normalize_key(value) when is_atom(value), do: Atom.to_string(value) - defp normalize_key(value), do: to_string(value) - - defp resolve_path_value(:missing, default), do: default - defp resolve_path_value(nil, default), do: default - - defp resolve_path_value(value, default) when is_binary(value) do - case normalize_path_token(value) do - :missing -> - default - - path -> - path - |> String.trim() - |> preserve_command_name() - |> then(fn - "" -> default - resolved -> resolved - end) - end - end - - defp resolve_path_value(_value, default), do: default - - defp preserve_command_name(path) do + defp validate_semantics(settings) do cond do - uri_path?(path) -> - path + is_nil(settings.tracker.kind) -> + {:error, :missing_tracker_kind} - String.contains?(path, "/") or String.contains?(path, "\\") -> - Path.expand(path) + settings.tracker.kind not in ["linear", "memory"] -> + {:error, {:unsupported_tracker_kind, settings.tracker.kind}} - true -> - path - end - end + settings.tracker.kind == "linear" and not is_binary(settings.tracker.api_key) -> + {:error, :missing_linear_api_token} - defp uri_path?(path) do - String.match?(to_string(path), ~r/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//) - end + settings.tracker.kind == "linear" and not is_binary(settings.tracker.project_slug) -> + {:error, :missing_linear_project_slug} - defp resolve_env_value(:missing, fallback), do: fallback - defp resolve_env_value(nil, fallback), do: fallback - - defp resolve_env_value(value, fallback) when is_binary(value) do - trimmed = String.trim(value) - - case env_reference_name(trimmed) do - {:ok, env_name} -> - env_name - |> System.get_env() - |> then(fn - nil -> fallback - "" -> nil - env_value -> env_value - end) - - :error -> - trimmed + true -> + :ok end end - defp resolve_env_value(_value, fallback), do: fallback - - defp normalize_path_token(value) when is_binary(value) do - trimmed = String.trim(value) - - case env_reference_name(trimmed) do - {:ok, env_name} -> resolve_env_token(env_name) - :error -> trimmed - end - end + defp format_config_error(reason) do + case reason do + {:invalid_workflow_config, message} -> + "Invalid WORKFLOW.md config: #{message}" - defp env_reference_name("$" <> env_name) do - if String.match?(env_name, ~r/^[A-Za-z_][A-Za-z0-9_]*$/) do - {:ok, env_name} - else - :error - end - end + {:missing_workflow_file, path, raw_reason} -> + "Missing WORKFLOW.md at #{path}: #{inspect(raw_reason)}" - defp env_reference_name(_value), do: :error + {:workflow_parse_error, raw_reason} -> + "Failed to parse WORKFLOW.md: #{inspect(raw_reason)}" - defp resolve_env_token(value) do - case System.get_env(value) do - nil -> :missing - env_value -> env_value - end - end + :workflow_front_matter_not_a_map -> + "Failed to parse WORKFLOW.md: workflow front matter must decode to a map" - defp normalize_secret_value(value) when is_binary(value) do - case String.trim(value) do - "" -> nil - trimmed -> trimmed + other -> + "Invalid WORKFLOW.md config: #{inspect(other)}" end end - - defp normalize_secret_value(_value), do: nil end diff --git a/elixir/lib/symphony_elixir/config/schema.ex b/elixir/lib/symphony_elixir/config/schema.ex new file mode 100644 index 0000000000..dcb67f9879 --- /dev/null +++ b/elixir/lib/symphony_elixir/config/schema.ex @@ -0,0 +1,493 @@ +defmodule SymphonyElixir.Config.Schema do + @moduledoc false + + use Ecto.Schema + + import Ecto.Changeset + + @primary_key false + + @type t :: %__MODULE__{} + + defmodule StringOrMap do + @moduledoc false + @behaviour Ecto.Type + + @spec type() :: :map + def type, do: :map + + @spec embed_as(term()) :: :self + def embed_as(_format), do: :self + + @spec equal?(term(), term()) :: boolean() + def equal?(left, right), do: left == right + + @spec cast(term()) :: {:ok, String.t() | map()} | :error + def cast(value) when is_binary(value) or is_map(value), do: {:ok, value} + def cast(_value), do: :error + + @spec load(term()) :: {:ok, String.t() | map()} | :error + def load(value) when is_binary(value) or is_map(value), do: {:ok, value} + def load(_value), do: :error + + @spec dump(term()) :: {:ok, String.t() | map()} | :error + def dump(value) when is_binary(value) or is_map(value), do: {:ok, value} + def dump(_value), do: :error + end + + defmodule Tracker do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:kind, :string) + field(:endpoint, :string, default: "https://api.linear.app/graphql") + field(:api_key, :string) + field(:project_slug, :string) + field(:assignee, :string) + field(:active_states, {:array, :string}, default: ["Todo", "In Progress"]) + field(:terminal_states, {:array, :string}, default: ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"]) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast( + attrs, + [:kind, :endpoint, :api_key, :project_slug, :assignee, :active_states, :terminal_states], + empty_values: [] + ) + end + end + + defmodule Polling do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:interval_ms, :integer, default: 30_000) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:interval_ms], empty_values: []) + |> validate_number(:interval_ms, greater_than: 0) + end + end + + defmodule Workspace do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:root, :string, default: Path.join(System.tmp_dir!(), "symphony_workspaces")) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:root], empty_values: []) + end + end + + defmodule Agent do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + alias SymphonyElixir.Config.Schema + + @primary_key false + embedded_schema do + field(:max_concurrent_agents, :integer, default: 10) + field(:max_turns, :integer, default: 20) + field(:max_retry_backoff_ms, :integer, default: 300_000) + field(:max_concurrent_agents_by_state, :map, default: %{}) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast( + attrs, + [:max_concurrent_agents, :max_turns, :max_retry_backoff_ms, :max_concurrent_agents_by_state], + empty_values: [] + ) + |> validate_number(:max_concurrent_agents, greater_than: 0) + |> validate_number(:max_turns, greater_than: 0) + |> validate_number(:max_retry_backoff_ms, greater_than: 0) + |> update_change(:max_concurrent_agents_by_state, &Schema.normalize_state_limits/1) + |> Schema.validate_state_limits(:max_concurrent_agents_by_state) + end + end + + defmodule Codex do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:command, :string, default: "codex app-server") + + field(:approval_policy, StringOrMap, + default: %{ + "reject" => %{ + "sandbox_approval" => true, + "rules" => true, + "mcp_elicitations" => true + } + } + ) + + field(:thread_sandbox, :string, default: "workspace-write") + field(:turn_sandbox_policy, :map) + field(:turn_timeout_ms, :integer, default: 3_600_000) + field(:read_timeout_ms, :integer, default: 5_000) + field(:stall_timeout_ms, :integer, default: 300_000) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast( + attrs, + [ + :command, + :approval_policy, + :thread_sandbox, + :turn_sandbox_policy, + :turn_timeout_ms, + :read_timeout_ms, + :stall_timeout_ms + ], + empty_values: [] + ) + |> validate_required([:command]) + |> validate_number(:turn_timeout_ms, greater_than: 0) + |> validate_number(:read_timeout_ms, greater_than: 0) + |> validate_number(:stall_timeout_ms, greater_than_or_equal_to: 0) + end + end + + defmodule Hooks do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:after_create, :string) + field(:before_run, :string) + field(:after_run, :string) + field(:before_remove, :string) + field(:timeout_ms, :integer, default: 60_000) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:after_create, :before_run, :after_run, :before_remove, :timeout_ms], empty_values: []) + |> validate_number(:timeout_ms, greater_than: 0) + end + end + + defmodule Observability do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:dashboard_enabled, :boolean, default: true) + field(:refresh_ms, :integer, default: 1_000) + field(:render_interval_ms, :integer, default: 16) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:dashboard_enabled, :refresh_ms, :render_interval_ms], empty_values: []) + |> validate_number(:refresh_ms, greater_than: 0) + |> validate_number(:render_interval_ms, greater_than: 0) + end + end + + defmodule Server do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:port, :integer) + field(:host, :string, default: "127.0.0.1") + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:port, :host], empty_values: []) + |> validate_number(:port, greater_than_or_equal_to: 0) + end + end + + embedded_schema do + embeds_one(:tracker, Tracker, on_replace: :update, defaults_to_struct: true) + embeds_one(:polling, Polling, on_replace: :update, defaults_to_struct: true) + embeds_one(:workspace, Workspace, on_replace: :update, defaults_to_struct: true) + embeds_one(:agent, Agent, on_replace: :update, defaults_to_struct: true) + embeds_one(:codex, Codex, on_replace: :update, defaults_to_struct: true) + embeds_one(:hooks, Hooks, on_replace: :update, defaults_to_struct: true) + embeds_one(:observability, Observability, on_replace: :update, defaults_to_struct: true) + embeds_one(:server, Server, on_replace: :update, defaults_to_struct: true) + end + + @spec parse(map()) :: {:ok, %__MODULE__{}} | {:error, {:invalid_workflow_config, String.t()}} + def parse(config) when is_map(config) do + config + |> normalize_keys() + |> drop_nil_values() + |> changeset() + |> apply_action(:validate) + |> case do + {:ok, settings} -> + {:ok, finalize_settings(settings)} + + {:error, changeset} -> + {:error, {:invalid_workflow_config, format_errors(changeset)}} + end + end + + @spec resolve_turn_sandbox_policy(%__MODULE__{}, Path.t() | nil) :: map() + def resolve_turn_sandbox_policy(settings, workspace \\ nil) do + case settings.codex.turn_sandbox_policy do + %{} = policy -> + policy + + _ -> + default_turn_sandbox_policy(workspace || settings.workspace.root) + end + end + + @spec normalize_issue_state(String.t()) :: String.t() + def normalize_issue_state(state_name) when is_binary(state_name) do + String.downcase(state_name) + end + + @doc false + @spec normalize_state_limits(nil | map()) :: map() + def normalize_state_limits(nil), do: %{} + + def normalize_state_limits(limits) when is_map(limits) do + Enum.reduce(limits, %{}, fn {state_name, limit}, acc -> + Map.put(acc, normalize_issue_state(to_string(state_name)), limit) + end) + end + + @doc false + @spec validate_state_limits(Ecto.Changeset.t(), atom()) :: Ecto.Changeset.t() + def validate_state_limits(changeset, field) do + validate_change(changeset, field, fn ^field, limits -> + Enum.flat_map(limits, fn {state_name, limit} -> + cond do + to_string(state_name) == "" -> + [{field, "state names must not be blank"}] + + not is_integer(limit) or limit <= 0 -> + [{field, "limits must be positive integers"}] + + true -> + [] + end + end) + end) + end + + defp changeset(attrs) do + %__MODULE__{} + |> cast(attrs, []) + |> cast_embed(:tracker, with: &Tracker.changeset/2) + |> cast_embed(:polling, with: &Polling.changeset/2) + |> cast_embed(:workspace, with: &Workspace.changeset/2) + |> cast_embed(:agent, with: &Agent.changeset/2) + |> cast_embed(:codex, with: &Codex.changeset/2) + |> cast_embed(:hooks, with: &Hooks.changeset/2) + |> cast_embed(:observability, with: &Observability.changeset/2) + |> cast_embed(:server, with: &Server.changeset/2) + end + + defp finalize_settings(settings) do + tracker = %{ + settings.tracker + | api_key: resolve_secret_setting(settings.tracker.api_key, System.get_env("LINEAR_API_KEY")), + assignee: resolve_secret_setting(settings.tracker.assignee, System.get_env("LINEAR_ASSIGNEE")) + } + + workspace = %{ + settings.workspace + | root: resolve_path_value(settings.workspace.root, Path.join(System.tmp_dir!(), "symphony_workspaces")) + } + + codex = %{ + settings.codex + | approval_policy: normalize_keys(settings.codex.approval_policy), + turn_sandbox_policy: normalize_optional_map(settings.codex.turn_sandbox_policy) + } + + %{settings | tracker: tracker, workspace: workspace, codex: codex} + end + + defp normalize_keys(value) when is_map(value) do + Enum.reduce(value, %{}, fn {key, raw_value}, normalized -> + Map.put(normalized, normalize_key(key), normalize_keys(raw_value)) + end) + end + + defp normalize_keys(value) when is_list(value), do: Enum.map(value, &normalize_keys/1) + defp normalize_keys(value), do: value + + defp normalize_optional_map(nil), do: nil + defp normalize_optional_map(value) when is_map(value), do: normalize_keys(value) + + defp normalize_key(value) when is_atom(value), do: Atom.to_string(value) + defp normalize_key(value), do: to_string(value) + + defp drop_nil_values(value) when is_map(value) do + Enum.reduce(value, %{}, fn {key, nested}, acc -> + case drop_nil_values(nested) do + nil -> acc + normalized -> Map.put(acc, key, normalized) + end + end) + end + + defp drop_nil_values(value) when is_list(value), do: Enum.map(value, &drop_nil_values/1) + defp drop_nil_values(value), do: value + + defp resolve_secret_setting(nil, fallback), do: normalize_secret_value(fallback) + + defp resolve_secret_setting(value, fallback) when is_binary(value) do + case resolve_env_value(value, fallback) do + resolved when is_binary(resolved) -> normalize_secret_value(resolved) + resolved -> resolved + end + end + + defp resolve_path_value(value, default) when is_binary(value) do + case normalize_path_token(value) do + :missing -> + Path.expand(default) + + "" -> + Path.expand(default) + + path -> + Path.expand(path) + end + end + + defp resolve_env_value(value, fallback) when is_binary(value) do + case env_reference_name(value) do + {:ok, env_name} -> + case System.get_env(env_name) do + nil -> fallback + "" -> nil + env_value -> env_value + end + + :error -> + value + end + end + + defp normalize_path_token(value) when is_binary(value) do + case env_reference_name(value) do + {:ok, env_name} -> resolve_env_token(env_name) + :error -> value + end + end + + defp env_reference_name("$" <> env_name) do + if String.match?(env_name, ~r/^[A-Za-z_][A-Za-z0-9_]*$/) do + {:ok, env_name} + else + :error + end + end + + defp env_reference_name(_value), do: :error + + defp resolve_env_token(env_name) do + case System.get_env(env_name) do + nil -> :missing + env_value -> env_value + end + end + + defp normalize_secret_value(value) when is_binary(value) do + if value == "", do: nil, else: value + end + + defp normalize_secret_value(_value), do: nil + + defp default_turn_sandbox_policy(workspace) do + writable_root = + if is_binary(workspace) and workspace != "" do + Path.expand(workspace) + else + Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + end + + %{ + "type" => "workspaceWrite", + "writableRoots" => [writable_root], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + end + + defp format_errors(changeset) do + changeset + |> traverse_errors(&translate_error/1) + |> flatten_errors() + |> Enum.join(", ") + end + + defp flatten_errors(errors, prefix \\ nil) + + defp flatten_errors(errors, prefix) when is_map(errors) do + Enum.flat_map(errors, fn {key, value} -> + next_prefix = + case prefix do + nil -> to_string(key) + current -> current <> "." <> to_string(key) + end + + flatten_errors(value, next_prefix) + end) + end + + defp flatten_errors(errors, prefix) when is_list(errors) do + Enum.map(errors, &(prefix <> " " <> &1)) + end + + defp translate_error({message, options}) do + Enum.reduce(options, message, fn {key, value}, acc -> + String.replace(acc, "%{#{key}}", error_value_to_string(value)) + end) + end + + defp error_value_to_string(value) when is_atom(value), do: Atom.to_string(value) + defp error_value_to_string(value), do: inspect(value) +end diff --git a/elixir/lib/symphony_elixir/http_server.ex b/elixir/lib/symphony_elixir/http_server.ex index 47686e934f..5f947b880a 100644 --- a/elixir/lib/symphony_elixir/http_server.ex +++ b/elixir/lib/symphony_elixir/http_server.ex @@ -20,7 +20,7 @@ defmodule SymphonyElixir.HttpServer do def start_link(opts \\ []) do case Keyword.get(opts, :port, Config.server_port()) do port when is_integer(port) and port >= 0 -> - host = Keyword.get(opts, :host, Config.server_host()) + host = Keyword.get(opts, :host, Config.settings!().server.host) orchestrator = Keyword.get(opts, :orchestrator, Orchestrator) snapshot_timeout_ms = Keyword.get(opts, :snapshot_timeout_ms, 15_000) diff --git a/elixir/lib/symphony_elixir/linear/client.ex b/elixir/lib/symphony_elixir/linear/client.ex index ad8eee550d..0ff290fe7c 100644 --- a/elixir/lib/symphony_elixir/linear/client.ex +++ b/elixir/lib/symphony_elixir/linear/client.ex @@ -105,10 +105,11 @@ defmodule SymphonyElixir.Linear.Client do @spec fetch_candidate_issues() :: {:ok, [Issue.t()]} | {:error, term()} def fetch_candidate_issues do - project_slug = Config.linear_project_slug() + tracker = Config.settings!().tracker + project_slug = tracker.project_slug cond do - is_nil(Config.linear_api_token()) -> + is_nil(tracker.api_key) -> {:error, :missing_linear_api_token} is_nil(project_slug) -> @@ -116,7 +117,7 @@ defmodule SymphonyElixir.Linear.Client do true -> with {:ok, assignee_filter} <- routing_assignee_filter() do - do_fetch_by_states(project_slug, Config.linear_active_states(), assignee_filter) + do_fetch_by_states(project_slug, tracker.active_states, assignee_filter) end end end @@ -128,10 +129,11 @@ defmodule SymphonyElixir.Linear.Client do if normalized_states == [] do {:ok, []} else - project_slug = Config.linear_project_slug() + tracker = Config.settings!().tracker + project_slug = tracker.project_slug cond do - is_nil(Config.linear_api_token()) -> + is_nil(tracker.api_key) -> {:error, :missing_linear_api_token} is_nil(project_slug) -> @@ -325,7 +327,7 @@ defmodule SymphonyElixir.Linear.Client do end defp graphql_headers do - case Config.linear_api_token() do + case Config.settings!().tracker.api_key do nil -> {:error, :missing_linear_api_token} @@ -339,7 +341,7 @@ defmodule SymphonyElixir.Linear.Client do end defp post_graphql_request(payload, headers) do - Req.post(Config.linear_endpoint(), + Req.post(Config.settings!().tracker.endpoint, headers: headers, json: payload, connect_options: [timeout: 30_000] @@ -432,7 +434,7 @@ defmodule SymphonyElixir.Linear.Client do defp assignee_id(%{} = assignee), do: normalize_assignee_match_value(assignee["id"]) defp routing_assignee_filter do - case Config.linear_assignee() do + case Config.settings!().tracker.assignee do nil -> {:ok, nil} diff --git a/elixir/lib/symphony_elixir/orchestrator.ex b/elixir/lib/symphony_elixir/orchestrator.ex index a4dead129e..4286ce1f64 100644 --- a/elixir/lib/symphony_elixir/orchestrator.ex +++ b/elixir/lib/symphony_elixir/orchestrator.ex @@ -49,10 +49,11 @@ defmodule SymphonyElixir.Orchestrator do @impl true def init(_opts) do now_ms = System.monotonic_time(:millisecond) + config = Config.settings!() state = %State{ - poll_interval_ms: Config.poll_interval_ms(), - max_concurrent_agents: Config.max_concurrent_agents(), + poll_interval_ms: config.polling.interval_ms, + max_concurrent_agents: config.agent.max_concurrent_agents, next_poll_due_at_ms: now_ms, poll_check_in_progress: false, codex_totals: @empty_codex_totals, @@ -196,20 +197,8 @@ defmodule SymphonyElixir.Orchestrator do state - {:error, :missing_codex_command} -> - Logger.error("Codex command missing in WORKFLOW.md") - state - - {:error, {:invalid_codex_approval_policy, value}} -> - Logger.error("Invalid codex.approval_policy in WORKFLOW.md: #{inspect(value)}") - state - - {:error, {:invalid_codex_thread_sandbox, value}} -> - Logger.error("Invalid codex.thread_sandbox in WORKFLOW.md: #{inspect(value)}") - state - - {:error, {:invalid_codex_turn_sandbox_policy, reason}} -> - Logger.error("Invalid codex.turn_sandbox_policy in WORKFLOW.md: #{inspect(reason)}") + {:error, {:invalid_workflow_config, message}} -> + Logger.error("Invalid WORKFLOW.md config: #{message}") state {:error, {:missing_workflow_file, path, reason}} -> @@ -365,7 +354,7 @@ defmodule SymphonyElixir.Orchestrator do end defp reconcile_stalled_running_issues(%State{} = state) do - timeout_ms = Config.codex_stall_timeout_ms() + timeout_ms = Config.settings!().codex.stall_timeout_ms cond do timeout_ms <= 0 -> @@ -562,14 +551,14 @@ defmodule SymphonyElixir.Orchestrator do end defp terminal_state_set do - Config.linear_terminal_states() + Config.settings!().tracker.terminal_states |> Enum.map(&normalize_issue_state/1) |> Enum.filter(&(&1 != "")) |> MapSet.new() end defp active_state_set do - Config.linear_active_states() + Config.settings!().tracker.active_states |> Enum.map(&normalize_issue_state/1) |> Enum.filter(&(&1 != "")) |> MapSet.new() @@ -774,7 +763,7 @@ defmodule SymphonyElixir.Orchestrator do defp cleanup_issue_workspace(_identifier), do: :ok defp run_terminal_workspace_cleanup do - case Tracker.fetch_issues_by_states(Config.linear_terminal_states()) do + case Tracker.fetch_issues_by_states(Config.settings!().tracker.terminal_states) do {:ok, issues} -> issues |> Enum.each(fn @@ -828,7 +817,7 @@ defmodule SymphonyElixir.Orchestrator do defp failure_retry_delay(attempt) do max_delay_power = min(attempt - 1, 10) - min(@failure_retry_base_ms * (1 <<< max_delay_power), Config.max_retry_backoff_ms()) + min(@failure_retry_base_ms * (1 <<< max_delay_power), Config.settings!().agent.max_retry_backoff_ms) end defp normalize_retry_attempt(attempt) when is_integer(attempt) and attempt > 0, do: attempt @@ -877,7 +866,8 @@ defmodule SymphonyElixir.Orchestrator do defp available_slots(%State{} = state) do max( - (state.max_concurrent_agents || Config.max_concurrent_agents()) - map_size(state.running), + (state.max_concurrent_agents || Config.settings!().agent.max_concurrent_agents) - + map_size(state.running), 0 ) end @@ -1098,10 +1088,12 @@ defmodule SymphonyElixir.Orchestrator do defp record_session_completion_totals(state, _running_entry), do: state defp refresh_runtime_config(%State{} = state) do + config = Config.settings!() + %{ state - | poll_interval_ms: Config.poll_interval_ms(), - max_concurrent_agents: Config.max_concurrent_agents() + | poll_interval_ms: config.polling.interval_ms, + max_concurrent_agents: config.agent.max_concurrent_agents } end diff --git a/elixir/lib/symphony_elixir/status_dashboard.ex b/elixir/lib/symphony_elixir/status_dashboard.ex index 19b628bfac..8cd4b17f36 100644 --- a/elixir/lib/symphony_elixir/status_dashboard.ex +++ b/elixir/lib/symphony_elixir/status_dashboard.ex @@ -99,10 +99,11 @@ defmodule SymphonyElixir.StatusDashboard do refresh_ms_override = keyword_override(opts, :refresh_ms) enabled_override = keyword_override(opts, :enabled) render_interval_ms_override = keyword_override(opts, :render_interval_ms) - refresh_ms = refresh_ms_override || Config.observability_refresh_ms() - render_interval_ms = render_interval_ms_override || Config.observability_render_interval_ms() + observability = Config.settings!().observability + refresh_ms = refresh_ms_override || observability.refresh_ms + render_interval_ms = render_interval_ms_override || observability.render_interval_ms render_fun = Keyword.get(opts, :render_fun, &render_to_terminal/1) - enabled = resolve_override(enabled_override, Config.observability_enabled?() and dashboard_enabled?()) + enabled = resolve_override(enabled_override, observability.dashboard_enabled and dashboard_enabled?()) schedule_tick(refresh_ms, enabled) {:ok, @@ -176,11 +177,13 @@ defmodule SymphonyElixir.StatusDashboard do def handle_info(:tick, state), do: {:noreply, state} defp refresh_runtime_config(%__MODULE__{} = state) do + observability = Config.settings!().observability + %{ state - | enabled: resolve_override(state.enabled_override, Config.observability_enabled?() and dashboard_enabled?()), - refresh_ms: state.refresh_ms_override || Config.observability_refresh_ms(), - render_interval_ms: state.render_interval_ms_override || Config.observability_render_interval_ms() + | enabled: resolve_override(state.enabled_override, observability.dashboard_enabled and dashboard_enabled?()), + refresh_ms: state.refresh_ms_override || observability.refresh_ms, + render_interval_ms: state.render_interval_ms_override || observability.render_interval_ms } end @@ -338,7 +341,7 @@ defmodule SymphonyElixir.StatusDashboard do codex_total_tokens = Map.get(codex_totals, :total_tokens, 0) codex_seconds_running = Map.get(codex_totals, :seconds_running, 0) agent_count = length(running) - max_agents = Config.max_concurrent_agents() + max_agents = Config.settings!().agent.max_concurrent_agents running_event_width = running_event_width(terminal_columns_override) running_rows = format_running_rows(running, running_event_width) running_to_backoff_spacer = if(running == [], do: [], else: ["│"]) @@ -391,7 +394,7 @@ defmodule SymphonyElixir.StatusDashboard do defp format_project_link_lines do project_part = - case Config.linear_project_slug() do + case Config.settings!().tracker.project_slug do project_slug when is_binary(project_slug) and project_slug != "" -> colorize(linear_project_url(project_slug), @ansi_cyan) @@ -427,7 +430,7 @@ defmodule SymphonyElixir.StatusDashboard do defp linear_project_url(project_slug), do: "https://linear.app/project/#{project_slug}/issues" defp dashboard_url do - dashboard_url(Config.server_host(), Config.server_port(), HttpServer.bound_port()) + dashboard_url(Config.settings!().server.host, Config.server_port(), HttpServer.bound_port()) end defp dashboard_url(_host, nil, _bound_port), do: nil diff --git a/elixir/lib/symphony_elixir/tracker.ex b/elixir/lib/symphony_elixir/tracker.ex index 504b54af30..000b6edf8d 100644 --- a/elixir/lib/symphony_elixir/tracker.ex +++ b/elixir/lib/symphony_elixir/tracker.ex @@ -38,7 +38,7 @@ defmodule SymphonyElixir.Tracker do @spec adapter() :: module() def adapter do - case Config.tracker_kind() do + case Config.settings!().tracker.kind do "memory" -> SymphonyElixir.Tracker.Memory _ -> SymphonyElixir.Linear.Adapter end diff --git a/elixir/lib/symphony_elixir/workspace.ex b/elixir/lib/symphony_elixir/workspace.ex index e8c085ea56..2401573f7f 100644 --- a/elixir/lib/symphony_elixir/workspace.ex +++ b/elixir/lib/symphony_elixir/workspace.ex @@ -71,7 +71,7 @@ defmodule SymphonyElixir.Workspace do @spec remove_issue_workspaces(term()) :: :ok def remove_issue_workspaces(identifier) when is_binary(identifier) do safe_id = safe_identifier(identifier) - workspace = Path.join(Config.workspace_root(), safe_id) + workspace = Path.join(Config.settings!().workspace.root, safe_id) remove(workspace) :ok @@ -84,8 +84,9 @@ defmodule SymphonyElixir.Workspace do @spec run_before_run_hook(Path.t(), map() | String.t() | nil) :: :ok | {:error, term()} def run_before_run_hook(workspace, issue_or_identifier) when is_binary(workspace) do issue_context = issue_context(issue_or_identifier) + hooks = Config.settings!().hooks - case Config.workspace_hooks()[:before_run] do + case hooks.before_run do nil -> :ok @@ -97,8 +98,9 @@ defmodule SymphonyElixir.Workspace do @spec run_after_run_hook(Path.t(), map() | String.t() | nil) :: :ok def run_after_run_hook(workspace, issue_or_identifier) when is_binary(workspace) do issue_context = issue_context(issue_or_identifier) + hooks = Config.settings!().hooks - case Config.workspace_hooks()[:after_run] do + case hooks.after_run do nil -> :ok @@ -109,7 +111,7 @@ defmodule SymphonyElixir.Workspace do end defp workspace_path_for_issue(safe_id) when is_binary(safe_id) do - Path.join(Config.workspace_root(), safe_id) + Path.join(Config.settings!().workspace.root, safe_id) end defp safe_identifier(identifier) do @@ -123,9 +125,11 @@ defmodule SymphonyElixir.Workspace do end defp maybe_run_after_create_hook(workspace, issue_context, created?) do + hooks = Config.settings!().hooks + case created? do true -> - case Config.workspace_hooks()[:after_create] do + case hooks.after_create do nil -> :ok @@ -139,9 +143,11 @@ defmodule SymphonyElixir.Workspace do end defp maybe_run_before_remove_hook(workspace) do + hooks = Config.settings!().hooks + case File.dir?(workspace) do true -> - case Config.workspace_hooks()[:before_remove] do + case hooks.before_remove do nil -> :ok @@ -164,7 +170,7 @@ defmodule SymphonyElixir.Workspace do defp ignore_hook_failure({:error, _reason}), do: :ok defp run_hook(command, workspace, issue_context, hook_name) do - timeout_ms = Config.workspace_hooks()[:timeout_ms] + timeout_ms = Config.settings!().hooks.timeout_ms Logger.info("Running workspace hook hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace}") @@ -212,7 +218,7 @@ defmodule SymphonyElixir.Workspace do defp validate_workspace_path(workspace) when is_binary(workspace) do expanded_workspace = Path.expand(workspace) - root = Path.expand(Config.workspace_root()) + root = Path.expand(Config.settings!().workspace.root) root_prefix = root <> "/" cond do diff --git a/elixir/lib/symphony_elixir_web/presenter.ex b/elixir/lib/symphony_elixir_web/presenter.ex index 34eb1e6641..dc78ab3234 100644 --- a/elixir/lib/symphony_elixir_web/presenter.ex +++ b/elixir/lib/symphony_elixir_web/presenter.ex @@ -66,7 +66,7 @@ defmodule SymphonyElixirWeb.Presenter do issue_id: issue_id_from_entries(running, retry), status: issue_status(running, retry), workspace: %{ - path: Path.join(Config.workspace_root(), issue_identifier) + path: Path.join(Config.settings!().workspace.root, issue_identifier) }, attempts: %{ restart_count: restart_count(retry), diff --git a/elixir/mix.exs b/elixir/mix.exs index 062706aab7..aff9e4d98c 100644 --- a/elixir/mix.exs +++ b/elixir/mix.exs @@ -73,7 +73,7 @@ defmodule SymphonyElixir.MixProject do {:jason, "~> 1.4"}, {:yaml_elixir, "~> 2.12"}, {:solid, "~> 1.2"}, - {:nimble_options, "~> 1.1"}, + {:ecto, "~> 3.13"}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev], runtime: false} ] diff --git a/elixir/mix.lock b/elixir/mix.lock index 4f52fd7003..f2f7c58d1c 100644 --- a/elixir/mix.lock +++ b/elixir/mix.lock @@ -6,6 +6,7 @@ "date_time_parser": {:hex, :date_time_parser, "1.3.0", "6ba16850b5ab83dd126576451023ab65349e29af2336ca5084aa1e37025b476e", [:mix], [{:kday, "~> 1.0", [hex: :kday, repo: "hexpm", optional: false]}], "hexpm", "93c8203a8ddc66b1f1531fc0e046329bf0b250c75ffa09567ef03d2c09218e8c"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, + "ecto": {:hex, :ecto, "3.13.5", "9d4a69700183f33bf97208294768e561f5c7f1ecf417e0fa1006e4a91713a834", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df9efebf70cf94142739ba357499661ef5dbb559ef902b68ea1f3c1fabce36de"}, "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, diff --git a/elixir/test/symphony_elixir/core_test.exs b/elixir/test/symphony_elixir/core_test.exs index 400c006e4f..023305eb72 100644 --- a/elixir/test/symphony_elixir/core_test.exs +++ b/elixir/test/symphony_elixir/core_test.exs @@ -11,26 +11,35 @@ defmodule SymphonyElixir.CoreTest do codex_command: nil ) - assert Config.poll_interval_ms() == 30_000 - assert Config.linear_active_states() == ["Todo", "In Progress"] - assert Config.linear_terminal_states() == ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"] - assert Config.linear_assignee() == nil - assert Config.agent_max_turns() == 20 + config = Config.settings!() + assert config.polling.interval_ms == 30_000 + assert config.tracker.active_states == ["Todo", "In Progress"] + assert config.tracker.terminal_states == ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"] + assert config.tracker.assignee == nil + assert config.agent.max_turns == 20 write_workflow_file!(Workflow.workflow_file_path(), poll_interval_ms: "invalid") - assert Config.poll_interval_ms() == 30_000 + + assert_raise ArgumentError, ~r/interval_ms/, fn -> + Config.settings!().polling.interval_ms + end + + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "polling.interval_ms" write_workflow_file!(Workflow.workflow_file_path(), poll_interval_ms: 45_000) - assert Config.poll_interval_ms() == 45_000 + assert Config.settings!().polling.interval_ms == 45_000 write_workflow_file!(Workflow.workflow_file_path(), max_turns: 0) - assert Config.agent_max_turns() == 20 + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "agent.max_turns" write_workflow_file!(Workflow.workflow_file_path(), max_turns: 5) - assert Config.agent_max_turns() == 5 + assert Config.settings!().agent.max_turns == 5 write_workflow_file!(Workflow.workflow_file_path(), tracker_active_states: "Todo, Review,") - assert Config.linear_active_states() == ["Todo", "Review"] + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "tracker.active_states" write_workflow_file!(Workflow.workflow_file_path(), tracker_api_token: "token", @@ -44,7 +53,13 @@ defmodule SymphonyElixir.CoreTest do codex_command: "" ) + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.command" + assert message =~ "can't be blank" + + write_workflow_file!(Workflow.workflow_file_path(), codex_command: " ") assert :ok = Config.validate!() + assert Config.settings!().codex.command == " " write_workflow_file!(Workflow.workflow_file_path(), codex_command: "/bin/sh app-server") assert :ok = Config.validate!() @@ -62,12 +77,14 @@ defmodule SymphonyElixir.CoreTest do assert :ok = Config.validate!() write_workflow_file!(Workflow.workflow_file_path(), codex_approval_policy: 123) - assert {:error, {:invalid_codex_approval_policy, 123}} = Config.validate!() + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.approval_policy" write_workflow_file!(Workflow.workflow_file_path(), codex_thread_sandbox: 123) - assert {:error, {:invalid_codex_thread_sandbox, 123}} = Config.validate!() + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.thread_sandbox" - write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: 123) + write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: "123") assert {:error, {:unsupported_tracker_kind, "123"}} = Config.validate!() end @@ -111,8 +128,8 @@ defmodule SymphonyElixir.CoreTest do codex_command: "/bin/sh app-server" ) - assert Config.linear_api_token() == env_api_key - assert Config.linear_project_slug() == "project" + assert Config.settings!().tracker.api_key == env_api_key + assert Config.settings!().tracker.project_slug == "project" assert :ok = Config.validate!() end @@ -129,7 +146,7 @@ defmodule SymphonyElixir.CoreTest do codex_command: "/bin/sh app-server" ) - assert Config.linear_assignee() == env_assignee + assert Config.settings!().tracker.assignee == env_assignee end test "workflow file path defaults to WORKFLOW.md in the current working directory when app env is unset" do diff --git a/elixir/test/symphony_elixir/extensions_test.exs b/elixir/test/symphony_elixir/extensions_test.exs index 59c8d0580b..27b2416b7b 100644 --- a/elixir/test/symphony_elixir/extensions_test.exs +++ b/elixir/test/symphony_elixir/extensions_test.exs @@ -187,7 +187,7 @@ defmodule SymphonyElixir.ExtensionsTest do Application.put_env(:symphony_elixir, :memory_tracker_recipient, self()) write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: "memory") - assert Config.tracker_kind() == "memory" + assert Config.settings!().tracker.kind == "memory" assert SymphonyElixir.Tracker.adapter() == Memory assert {:ok, [^issue]} = SymphonyElixir.Tracker.fetch_candidate_issues() assert {:ok, [^issue]} = SymphonyElixir.Tracker.fetch_issues_by_states([" in progress ", 42]) @@ -382,7 +382,7 @@ defmodule SymphonyElixir.ExtensionsTest do "issue_identifier" => "MT-HTTP", "issue_id" => "issue-http", "status" => "running", - "workspace" => %{"path" => Path.join(Config.workspace_root(), "MT-HTTP")}, + "workspace" => %{"path" => Path.join(Config.settings!().workspace.root, "MT-HTTP")}, "attempts" => %{"restart_count" => 0, "current_retry_attempt" => 0}, "running" => %{ "session_id" => "thread-http", diff --git a/elixir/test/symphony_elixir/workspace_and_config_test.exs b/elixir/test/symphony_elixir/workspace_and_config_test.exs index 10f9f524a5..7b681d54f5 100644 --- a/elixir/test/symphony_elixir/workspace_and_config_test.exs +++ b/elixir/test/symphony_elixir/workspace_and_config_test.exs @@ -1,5 +1,8 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do use SymphonyElixir.TestSupport + alias Ecto.Changeset + alias SymphonyElixir.Config.Schema + alias SymphonyElixir.Config.Schema.{Codex, StringOrMap} alias SymphonyElixir.Linear.Client test "workspace bootstrap can be implemented in after_create hook" do @@ -533,8 +536,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do hook_before_remove: "echo before_remove > \"#{before_remove_marker}\"" ) - assert Config.workspace_hooks().after_create =~ "echo after_create > after_create.log" - assert Config.workspace_hooks().before_remove =~ "echo before_remove >" + config = Config.settings!() + assert config.hooks.after_create =~ "echo after_create > after_create.log" + assert config.hooks.before_remove =~ "echo before_remove >" assert {:ok, workspace} = Workspace.create_for_issue("MT-HOOKS") assert File.read!(Path.join(workspace, "after_create.log")) == "after_create\n" @@ -655,14 +659,15 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do tracker_project_slug: nil ) - assert Config.linear_endpoint() == "https://api.linear.app/graphql" - assert Config.linear_api_token() == nil - assert Config.linear_project_slug() == nil - assert Config.workspace_root() == Path.join(System.tmp_dir!(), "symphony_workspaces") - assert Config.max_concurrent_agents() == 10 - assert Config.codex_command() == "codex app-server" + config = Config.settings!() + assert config.tracker.endpoint == "https://api.linear.app/graphql" + assert config.tracker.api_key == nil + assert config.tracker.project_slug == nil + assert config.workspace.root == Path.join(System.tmp_dir!(), "symphony_workspaces") + assert config.agent.max_concurrent_agents == 10 + assert config.codex.command == "codex app-server" - assert Config.codex_approval_policy() == %{ + assert config.codex.approval_policy == %{ "reject" => %{ "sandbox_approval" => true, "rules" => true, @@ -670,7 +675,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do } } - assert Config.codex_thread_sandbox() == "workspace-write" + assert config.codex.thread_sandbox == "workspace-write" assert Config.codex_turn_sandbox_policy() == %{ "type" => "workspaceWrite", @@ -681,12 +686,12 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do "excludeSlashTmp" => false } - assert Config.codex_turn_timeout_ms() == 3_600_000 - assert Config.codex_read_timeout_ms() == 5_000 - assert Config.codex_stall_timeout_ms() == 300_000 + assert config.codex.turn_timeout_ms == 3_600_000 + assert config.codex.read_timeout_ms == 5_000 + assert config.codex.stall_timeout_ms == 300_000 write_workflow_file!(Workflow.workflow_file_path(), codex_command: "codex app-server --model gpt-5.3-codex") - assert Config.codex_command() == "codex app-server --model gpt-5.3-codex" + assert Config.settings!().codex.command == "codex app-server --model gpt-5.3-codex" write_workflow_file!(Workflow.workflow_file_path(), codex_approval_policy: "on-request", @@ -694,8 +699,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do codex_turn_sandbox_policy: %{type: "workspaceWrite", writableRoots: ["/tmp/workspace", "/tmp/cache"]} ) - assert Config.codex_approval_policy() == "on-request" - assert Config.codex_thread_sandbox() == "workspace-write" + config = Config.settings!() + assert config.codex.approval_policy == "on-request" + assert config.codex.thread_sandbox == "workspace-write" assert Config.codex_turn_sandbox_policy() == %{ "type" => "workspaceWrite", @@ -703,19 +709,24 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do } write_workflow_file!(Workflow.workflow_file_path(), tracker_active_states: ",") - assert Config.linear_active_states() == ["Todo", "In Progress"] + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "tracker.active_states" write_workflow_file!(Workflow.workflow_file_path(), max_concurrent_agents: "bad") - assert Config.max_concurrent_agents() == 10 + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "agent.max_concurrent_agents" write_workflow_file!(Workflow.workflow_file_path(), codex_turn_timeout_ms: "bad") - assert Config.codex_turn_timeout_ms() == 3_600_000 + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.turn_timeout_ms" write_workflow_file!(Workflow.workflow_file_path(), codex_read_timeout_ms: "bad") - assert Config.codex_read_timeout_ms() == 5_000 + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.read_timeout_ms" write_workflow_file!(Workflow.workflow_file_path(), codex_stall_timeout_ms: "bad") - assert Config.codex_stall_timeout_ms() == 300_000 + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.stall_timeout_ms" write_workflow_file!(Workflow.workflow_file_path(), tracker_active_states: %{todo: true}, @@ -732,49 +743,19 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do server_host: 123 ) - assert Config.linear_active_states() == ["Todo", "In Progress"] - assert Config.linear_terminal_states() == ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"] - assert Config.poll_interval_ms() == 30_000 - assert Config.workspace_root() == Path.join(System.tmp_dir!(), "symphony_workspaces") - assert Config.max_retry_backoff_ms() == 300_000 - assert Config.max_concurrent_agents_for_state("Todo") == 1 - assert Config.max_concurrent_agents_for_state("Review") == 10 - assert Config.hook_timeout_ms() == 60_000 - assert Config.observability_enabled?() - assert Config.observability_refresh_ms() == 1_000 - assert Config.observability_render_interval_ms() == 16 - assert Config.server_port() == nil - assert Config.server_host() == "123" + assert {:error, {:invalid_workflow_config, _message}} = Config.validate!() write_workflow_file!(Workflow.workflow_file_path(), codex_approval_policy: "") - - assert Config.codex_approval_policy() == %{ - "reject" => %{ - "sandbox_approval" => true, - "rules" => true, - "mcp_elicitations" => true - } - } - - assert {:error, {:invalid_codex_approval_policy, ""}} = Config.validate!() + assert :ok = Config.validate!() + assert Config.settings!().codex.approval_policy == "" write_workflow_file!(Workflow.workflow_file_path(), codex_thread_sandbox: "") - assert Config.codex_thread_sandbox() == "workspace-write" - assert {:error, {:invalid_codex_thread_sandbox, ""}} = Config.validate!() + assert :ok = Config.validate!() + assert Config.settings!().codex.thread_sandbox == "" write_workflow_file!(Workflow.workflow_file_path(), codex_turn_sandbox_policy: "bad") - - assert Config.codex_turn_sandbox_policy() == %{ - "type" => "workspaceWrite", - "writableRoots" => [Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces"))], - "readOnlyAccess" => %{"type" => "fullAccess"}, - "networkAccess" => false, - "excludeTmpdirEnvVar" => false, - "excludeSlashTmp" => false - } - - assert {:error, {:invalid_codex_turn_sandbox_policy, {:unsupported_value, "bad"}}} = - Config.validate!() + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "codex.turn_sandbox_policy" write_workflow_file!(Workflow.workflow_file_path(), codex_approval_policy: "future-policy", @@ -785,8 +766,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do } ) - assert Config.codex_approval_policy() == "future-policy" - assert Config.codex_thread_sandbox() == "future-sandbox" + config = Config.settings!() + assert config.codex.approval_policy == "future-policy" + assert config.codex.thread_sandbox == "future-sandbox" assert Config.codex_turn_sandbox_policy() == %{ "type" => "futureSandbox", @@ -796,7 +778,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert :ok = Config.validate!() write_workflow_file!(Workflow.workflow_file_path(), codex_command: "codex app-server") - assert Config.codex_command() == "codex app-server" + assert Config.settings!().codex.command == "codex app-server" end test "config resolves $VAR references for env-backed secret and path values" do @@ -823,9 +805,10 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do codex_command: "#{codex_bin} app-server" ) - assert Config.linear_api_token() == api_key - assert Config.workspace_root() == Path.expand(workspace_root) - assert Config.codex_command() == "#{codex_bin} app-server" + config = Config.settings!() + assert config.tracker.api_key == api_key + assert config.workspace.root == Path.expand(workspace_root) + assert config.codex.command == "#{codex_bin} app-server" end test "config no longer resolves legacy env: references" do @@ -850,8 +833,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do workspace_root: "env:#{workspace_env_var}" ) - assert Config.linear_api_token() == "env:#{api_key_env_var}" - assert Config.workspace_root() == "env:#{workspace_env_var}" + config = Config.settings!() + assert config.tracker.api_key == "env:#{api_key_env_var}" + assert config.workspace.root == Path.expand("env:#{workspace_env_var}") end test "config supports per-state max concurrent agent overrides" do @@ -868,7 +852,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do File.write!(Workflow.workflow_file_path(), workflow) - assert Config.max_concurrent_agents() == 10 + assert Config.settings!().agent.max_concurrent_agents == 10 assert Config.max_concurrent_agents_for_state("Todo") == 1 assert Config.max_concurrent_agents_for_state("In Progress") == 4 assert Config.max_concurrent_agents_for_state("In Review") == 2 @@ -876,6 +860,122 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert Config.max_concurrent_agents_for_state(:not_a_string) == 10 end + test "schema helpers cover custom type and state limit validation" do + assert StringOrMap.type() == :map + assert StringOrMap.embed_as(:json) == :self + assert StringOrMap.equal?(%{"a" => 1}, %{"a" => 1}) + refute StringOrMap.equal?(%{"a" => 1}, %{"a" => 2}) + + assert {:ok, "value"} = StringOrMap.cast("value") + assert {:ok, %{"a" => 1}} = StringOrMap.cast(%{"a" => 1}) + assert :error = StringOrMap.cast(123) + + assert {:ok, "value"} = StringOrMap.load("value") + assert :error = StringOrMap.load(123) + + assert {:ok, %{"a" => 1}} = StringOrMap.dump(%{"a" => 1}) + assert :error = StringOrMap.dump(123) + + assert Schema.normalize_state_limits(nil) == %{} + + assert Schema.normalize_state_limits(%{"In Progress" => 2, todo: 1}) == %{ + "todo" => 1, + "in progress" => 2 + } + + changeset = + {%{}, %{limits: :map}} + |> Changeset.cast(%{limits: %{"" => 1, "todo" => 0}}, [:limits]) + |> Schema.validate_state_limits(:limits) + + assert changeset.errors == [ + limits: {"state names must not be blank", []}, + limits: {"limits must be positive integers", []} + ] + end + + test "schema parse normalizes policy keys and env-backed fallbacks" do + missing_workspace_env = "SYMP_MISSING_WORKSPACE_#{System.unique_integer([:positive])}" + empty_secret_env = "SYMP_EMPTY_SECRET_#{System.unique_integer([:positive])}" + missing_secret_env = "SYMP_MISSING_SECRET_#{System.unique_integer([:positive])}" + + previous_missing_workspace_env = System.get_env(missing_workspace_env) + previous_empty_secret_env = System.get_env(empty_secret_env) + previous_missing_secret_env = System.get_env(missing_secret_env) + previous_linear_api_key = System.get_env("LINEAR_API_KEY") + + System.delete_env(missing_workspace_env) + System.put_env(empty_secret_env, "") + System.delete_env(missing_secret_env) + System.put_env("LINEAR_API_KEY", "fallback-linear-token") + + on_exit(fn -> + restore_env(missing_workspace_env, previous_missing_workspace_env) + restore_env(empty_secret_env, previous_empty_secret_env) + restore_env(missing_secret_env, previous_missing_secret_env) + restore_env("LINEAR_API_KEY", previous_linear_api_key) + end) + + assert {:ok, settings} = + Schema.parse(%{ + tracker: %{api_key: "$#{empty_secret_env}"}, + workspace: %{root: "$#{missing_workspace_env}"}, + codex: %{approval_policy: %{reject: %{sandbox_approval: true}}} + }) + + assert settings.tracker.api_key == nil + assert settings.workspace.root == Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + + assert settings.codex.approval_policy == %{ + "reject" => %{"sandbox_approval" => true} + } + + assert {:ok, settings} = + Schema.parse(%{ + tracker: %{api_key: "$#{missing_secret_env}"}, + workspace: %{root: ""} + }) + + assert settings.tracker.api_key == "fallback-linear-token" + assert settings.workspace.root == Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + end + + test "schema resolves sandbox policies from explicit and default workspaces" do + explicit_policy = %{"type" => "workspaceWrite", "writableRoots" => ["/tmp/explicit"]} + + assert Schema.resolve_turn_sandbox_policy(%Schema{ + codex: %Codex{turn_sandbox_policy: explicit_policy}, + workspace: %Schema.Workspace{root: "/tmp/ignored"} + }) == explicit_policy + + assert Schema.resolve_turn_sandbox_policy(%Schema{ + codex: %Codex{turn_sandbox_policy: nil}, + workspace: %Schema.Workspace{root: ""} + }) == %{ + "type" => "workspaceWrite", + "writableRoots" => [Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces"))], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + + assert Schema.resolve_turn_sandbox_policy( + %Schema{ + codex: %Codex{turn_sandbox_policy: nil}, + workspace: %Schema.Workspace{root: "/tmp/ignored"} + }, + "/tmp/workspace" + ) == %{ + "type" => "workspaceWrite", + "writableRoots" => [Path.expand("/tmp/workspace")], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + end + test "workflow prompt is used when building base prompt" do workflow_prompt = "Workflow prompt body used as codex instruction." From 8afebb3919b35f8d3470c243eaff178190c10f04 Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Tue, 10 Mar 2026 11:20:31 -0700 Subject: [PATCH 02/41] Align workflow config spec with schema behavior #### Context The schema and tests now treat tracker state lists as YAML lists and only lowercase state-limit keys, but `SPEC.md` still documented older and stricter behavior. #### TL;DR *Align `SPEC.md` with the current workflow config and state normalization behavior.* #### Summary - document tracker `active_states` and `terminal_states` as YAML string lists - update the cheat sheet defaults to use list examples instead of CSV strings - remove trim-based state normalization language from the spec #### Alternatives - restore support for comma-separated tracker state strings in the schema - keep trim-based normalization in the spec and change the implementation #### Test Plan - [x] `make -C elixir all` - [x] Docs-only diff reviewed against current schema and tests --- SPEC.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/SPEC.md b/SPEC.md index 47d6abe725..e861f3b9e6 100644 --- a/SPEC.md +++ b/SPEC.md @@ -273,7 +273,7 @@ Fields: - Derive from `issue.identifier` by replacing any character not in `[A-Za-z0-9._-]` with `_`. - Use the sanitized value for the workspace directory name. - `Normalized Issue State` - - Compare states after `trim` + `lowercase`. + - Compare states after `lowercase`. - `Session ID` - Compose from coding-agent `thread_id` and `turn_id` as `-`. @@ -351,9 +351,9 @@ Fields: - If `$VAR_NAME` resolves to an empty string, treat the key as missing. - `project_slug` (string) - Required for dispatch when `tracker.kind == "linear"`. -- `active_states` (list of strings or comma-separated string) +- `active_states` (list of strings) - Default: `Todo`, `In Progress` -- `terminal_states` (list of strings or comma-separated string) +- `terminal_states` (list of strings) - Default: `Closed`, `Cancelled`, `Canceled`, `Duplicate`, `Done` #### 5.3.2 `polling` (object) @@ -410,7 +410,7 @@ Fields: - Changes should be re-applied at runtime and affect future retry scheduling. - `max_concurrent_agents_by_state` (map `state_name -> positive integer`) - Default: empty map. - - State keys are normalized (`trim` + `lowercase`) for lookup. + - State keys are normalized (`lowercase`) for lookup. - Invalid entries (non-positive or non-numeric) are ignored. #### 5.3.6 `codex` (object) @@ -555,8 +555,8 @@ This section is intentionally redundant so a coding agent can implement the conf - `tracker.endpoint`: string, default `https://api.linear.app/graphql` when `tracker.kind=linear` - `tracker.api_key`: string or `$VAR`, canonical env `LINEAR_API_KEY` when `tracker.kind=linear` - `tracker.project_slug`: string, required when `tracker.kind=linear` -- `tracker.active_states`: list/string, default `Todo, In Progress` -- `tracker.terminal_states`: list/string, default `Closed, Cancelled, Canceled, Duplicate, Done` +- `tracker.active_states`: list of strings, default `["Todo", "In Progress"]` +- `tracker.terminal_states`: list of strings, default `["Closed", "Cancelled", "Canceled", "Duplicate", "Done"]` - `polling.interval_ms`: integer, default `30000` - `workspace.root`: path, default `/symphony_workspaces` - `hooks.after_create`: shell script or null From c9ec3f15b9b38631a7dbb987d1c48e8a2aff7eba Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Tue, 10 Mar 2026 12:53:25 -0700 Subject: [PATCH 03/41] Stabilize Symphony Elixir orchestration and policy handling #### Context Symphony Elixir had a few correctness gaps around retry scheduling, refresh coalescing, workspace path handling, Codex turn policy forwarding, and running-issue refreshes. #### TL;DR *Stabilize Symphony Elixir orchestration, workspace safety, and Codex policy passthrough.* #### Summary - Ignore stale retry timers and coalesce superseded poll ticks in the orchestrator. - Canonicalize workspace paths and clarify last-known-good workflow reload behavior. - Pass explicit Codex turn sandbox policies through unchanged and add integration coverage. - Paginate Linear by-id issue-state refreshes so running issue reconciliation sees all IDs. #### Alternatives - Keep local validation and rewriting of explicit turn sandbox policies, but that drifted from Codex semantics and broke documented policy types. - Keep single-request by-id issue refreshes, but that truncates after 50 issues and can stop healthy workers during reconciliation. #### Test Plan - [x] `make -C elixir all` - [x] Targeted regression coverage for AppServer turn policy passthrough and Linear by-id pagination --- elixir/README.md | 11 +- elixir/lib/symphony_elixir/agent_runner.ex | 2 +- .../lib/symphony_elixir/codex/app_server.ex | 37 +-- elixir/lib/symphony_elixir/config.ex | 24 +- elixir/lib/symphony_elixir/config/schema.ex | 24 ++ elixir/lib/symphony_elixir/linear/client.ex | 62 ++++- elixir/lib/symphony_elixir/orchestrator.ex | 116 +++++++-- elixir/lib/symphony_elixir/path_safety.ex | 50 ++++ elixir/lib/symphony_elixir/workspace.ex | 69 +++--- .../test/symphony_elixir/app_server_test.exs | 144 ++++++++++++ elixir/test/symphony_elixir/core_test.exs | 165 ++++++++++++- .../workspace_and_config_test.exs | 221 +++++++++++++++++- 12 files changed, 817 insertions(+), 108 deletions(-) create mode 100644 elixir/lib/symphony_elixir/path_safety.ex diff --git a/elixir/README.md b/elixir/README.md index 3f711587ce..004b5943a4 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -14,7 +14,7 @@ This directory contains the current Elixir/OTP implementation of Symphony, based ## How it works 1. Polls Linear for candidate work -2. Creates an isolated workspace per issue +2. Creates a workspace per issue 3. Launches Codex in [App Server mode](https://developers.openai.com/codex/app-server/) inside the workspace 4. Sends a workflow prompt to Codex @@ -116,8 +116,9 @@ Notes: - `codex.turn_sandbox_policy` defaults to a `workspaceWrite` policy rooted at the current issue workspace - Supported `codex.approval_policy` values depend on the targeted Codex app-server version. In the current local Codex schema, string values include `untrusted`, `on-failure`, `on-request`, and `never`, and object-form `reject` is also supported. - Supported `codex.thread_sandbox` values: `read-only`, `workspace-write`, `danger-full-access`. -- Supported `codex.turn_sandbox_policy.type` values: `dangerFullAccess`, `readOnly`, - `externalSandbox`, `workspaceWrite`. +- When `codex.turn_sandbox_policy` is set explicitly, Symphony passes the map through to Codex + unchanged. Compatibility then depends on the targeted Codex app-server version rather than local + Symphony validation. - `agent.max_turns` caps how many back-to-back Codex turns Symphony will run in a single agent invocation when a turn completes normally but the issue is still in an active state. Default: `20`. - If the Markdown body is blank, Symphony uses a default prompt template that includes the issue @@ -144,7 +145,9 @@ codex: command: "$CODEX_BIN app-server --model gpt-5.3-codex" ``` -- If `WORKFLOW.md` is missing or has invalid YAML, startup and scheduling are halted until fixed. +- If `WORKFLOW.md` is missing or has invalid YAML at startup, Symphony does not boot. +- If a later reload fails, Symphony keeps running with the last known good workflow and logs the + reload error until the file is fixed. - `server.port` or CLI `--port` enables the optional Phoenix LiveView dashboard and JSON API at `/`, `/api/v1/state`, `/api/v1/`, and `/api/v1/refresh`. diff --git a/elixir/lib/symphony_elixir/agent_runner.ex b/elixir/lib/symphony_elixir/agent_runner.ex index 146350602c..482d547685 100644 --- a/elixir/lib/symphony_elixir/agent_runner.ex +++ b/elixir/lib/symphony_elixir/agent_runner.ex @@ -1,6 +1,6 @@ defmodule SymphonyElixir.AgentRunner do @moduledoc """ - Executes a single Linear issue in an isolated workspace with Codex. + Executes a single Linear issue in its workspace with Codex. """ require Logger diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index 1f9252e2f1..ef79022417 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -4,7 +4,7 @@ defmodule SymphonyElixir.Codex.AppServer do """ require Logger - alias SymphonyElixir.{Codex.DynamicTool, Config} + alias SymphonyElixir.{Codex.DynamicTool, Config, PathSafety} @initialize_id 1 @thread_start_id 2 @@ -37,10 +37,9 @@ defmodule SymphonyElixir.Codex.AppServer do @spec start_session(Path.t()) :: {:ok, session()} | {:error, term()} def start_session(workspace) do - with :ok <- validate_workspace_cwd(workspace), - {:ok, port} <- start_port(workspace) do + with {:ok, expanded_workspace} <- validate_workspace_cwd(workspace), + {:ok, port} <- start_port(expanded_workspace) do metadata = port_metadata(port) - expanded_workspace = Path.expand(workspace) with {:ok, session_policies} <- session_policies(expanded_workspace), {:ok, thread_id} <- do_start_session(port, expanded_workspace, session_policies) do @@ -142,20 +141,30 @@ defmodule SymphonyElixir.Codex.AppServer do end defp validate_workspace_cwd(workspace) when is_binary(workspace) do - workspace_path = Path.expand(workspace) - workspace_root = Path.expand(Config.settings!().workspace.root) + expanded_workspace = Path.expand(workspace) + expanded_root = Path.expand(Config.settings!().workspace.root) + expanded_root_prefix = expanded_root <> "/" - root_prefix = workspace_root <> "/" + with {:ok, canonical_workspace} <- PathSafety.canonicalize(expanded_workspace), + {:ok, canonical_root} <- PathSafety.canonicalize(expanded_root) do + canonical_root_prefix = canonical_root <> "/" - cond do - workspace_path == workspace_root -> - {:error, {:invalid_workspace_cwd, :workspace_root, workspace_path}} + cond do + canonical_workspace == canonical_root -> + {:error, {:invalid_workspace_cwd, :workspace_root, canonical_workspace}} - not String.starts_with?(workspace_path <> "/", root_prefix) -> - {:error, {:invalid_workspace_cwd, :outside_workspace_root, workspace_path, workspace_root}} + String.starts_with?(canonical_workspace <> "/", canonical_root_prefix) -> + {:ok, canonical_workspace} - true -> - :ok + String.starts_with?(expanded_workspace <> "/", expanded_root_prefix) -> + {:error, {:invalid_workspace_cwd, :symlink_escape, expanded_workspace, canonical_root}} + + true -> + {:error, {:invalid_workspace_cwd, :outside_workspace_root, canonical_workspace, canonical_root}} + end + else + {:error, {:path_canonicalize_failed, path, reason}} -> + {:error, {:invalid_workspace_cwd, :path_unreadable, path, reason}} end end diff --git a/elixir/lib/symphony_elixir/config.ex b/elixir/lib/symphony_elixir/config.ex index 84623ca23c..87011ac134 100644 --- a/elixir/lib/symphony_elixir/config.ex +++ b/elixir/lib/symphony_elixir/config.ex @@ -63,8 +63,13 @@ defmodule SymphonyElixir.Config do @spec codex_turn_sandbox_policy(Path.t() | nil) :: map() def codex_turn_sandbox_policy(workspace \\ nil) do - settings!() - |> Schema.resolve_turn_sandbox_policy(workspace) + case Schema.resolve_runtime_turn_sandbox_policy(settings!(), workspace) do + {:ok, policy} -> + policy + + {:error, reason} -> + raise ArgumentError, message: "Invalid codex turn sandbox policy: #{inspect(reason)}" + end end @spec workflow_prompt() :: String.t() @@ -96,12 +101,15 @@ defmodule SymphonyElixir.Config do @spec codex_runtime_settings(Path.t() | nil) :: {:ok, codex_runtime_settings()} | {:error, term()} def codex_runtime_settings(workspace \\ nil) do with {:ok, settings} <- settings() do - {:ok, - %{ - approval_policy: settings.codex.approval_policy, - thread_sandbox: settings.codex.thread_sandbox, - turn_sandbox_policy: Schema.resolve_turn_sandbox_policy(settings, workspace) - }} + with {:ok, turn_sandbox_policy} <- + Schema.resolve_runtime_turn_sandbox_policy(settings, workspace) do + {:ok, + %{ + approval_policy: settings.codex.approval_policy, + thread_sandbox: settings.codex.thread_sandbox, + turn_sandbox_policy: turn_sandbox_policy + }} + end end end diff --git a/elixir/lib/symphony_elixir/config/schema.ex b/elixir/lib/symphony_elixir/config/schema.ex index dcb67f9879..c9ad948f97 100644 --- a/elixir/lib/symphony_elixir/config/schema.ex +++ b/elixir/lib/symphony_elixir/config/schema.ex @@ -5,6 +5,8 @@ defmodule SymphonyElixir.Config.Schema do import Ecto.Changeset + alias SymphonyElixir.PathSafety + @primary_key false @type t :: %__MODULE__{} @@ -278,6 +280,18 @@ defmodule SymphonyElixir.Config.Schema do end end + @spec resolve_runtime_turn_sandbox_policy(%__MODULE__{}, Path.t() | nil) :: + {:ok, map()} | {:error, term()} + def resolve_runtime_turn_sandbox_policy(settings, workspace \\ nil) do + case settings.codex.turn_sandbox_policy do + %{} = policy -> + {:ok, policy} + + _ -> + default_runtime_turn_sandbox_policy(workspace || settings.workspace.root) + end + end + @spec normalize_issue_state(String.t()) :: String.t() def normalize_issue_state(state_name) when is_binary(state_name) do String.downcase(state_name) @@ -457,6 +471,16 @@ defmodule SymphonyElixir.Config.Schema do } end + defp default_runtime_turn_sandbox_policy(workspace_root) when is_binary(workspace_root) do + with {:ok, canonical_workspace_root} <- PathSafety.canonicalize(workspace_root) do + {:ok, default_turn_sandbox_policy(canonical_workspace_root)} + end + end + + defp default_runtime_turn_sandbox_policy(workspace_root) do + {:error, {:unsafe_turn_sandbox_policy, {:invalid_workspace_root, workspace_root}}} + end + defp format_errors(changeset) do changeset |> traverse_errors(&translate_error/1) diff --git a/elixir/lib/symphony_elixir/linear/client.ex b/elixir/lib/symphony_elixir/linear/client.ex index 0ff290fe7c..1a19649660 100644 --- a/elixir/lib/symphony_elixir/linear/client.ex +++ b/elixir/lib/symphony_elixir/linear/client.ex @@ -220,6 +220,22 @@ defmodule SymphonyElixir.Linear.Client do |> finalize_paginated_issues() end + @doc false + @spec fetch_issue_states_by_ids_for_test([String.t()], (String.t(), map() -> {:ok, map()} | {:error, term()})) :: + {:ok, [Issue.t()]} | {:error, term()} + def fetch_issue_states_by_ids_for_test(issue_ids, graphql_fun) + when is_list(issue_ids) and is_function(graphql_fun, 2) do + ids = Enum.uniq(issue_ids) + + case ids do + [] -> + {:ok, []} + + ids -> + do_fetch_issue_states(ids, nil, graphql_fun) + end + end + defp do_fetch_by_states(project_slug, state_names, assignee_filter) do do_fetch_by_states_page(project_slug, state_names, assignee_filter, nil, []) end @@ -256,19 +272,57 @@ defmodule SymphonyElixir.Linear.Client do defp finalize_paginated_issues(acc_issues) when is_list(acc_issues), do: Enum.reverse(acc_issues) defp do_fetch_issue_states(ids, assignee_filter) do - case graphql(@query_by_ids, %{ - ids: ids, - first: Enum.min([length(ids), @issue_page_size]), + do_fetch_issue_states(ids, assignee_filter, &graphql/2) + end + + defp do_fetch_issue_states(ids, assignee_filter, graphql_fun) + when is_list(ids) and is_function(graphql_fun, 2) do + issue_order_index = issue_order_index(ids) + do_fetch_issue_states_page(ids, assignee_filter, graphql_fun, [], issue_order_index) + end + + defp do_fetch_issue_states_page([], _assignee_filter, _graphql_fun, acc_issues, issue_order_index) do + acc_issues + |> finalize_paginated_issues() + |> sort_issues_by_requested_ids(issue_order_index) + |> then(&{:ok, &1}) + end + + defp do_fetch_issue_states_page(ids, assignee_filter, graphql_fun, acc_issues, issue_order_index) do + {batch_ids, rest_ids} = Enum.split(ids, @issue_page_size) + + case graphql_fun.(@query_by_ids, %{ + ids: batch_ids, + first: length(batch_ids), relationFirst: @issue_page_size }) do {:ok, body} -> - decode_linear_response(body, assignee_filter) + with {:ok, issues} <- decode_linear_response(body, assignee_filter) do + updated_acc = prepend_page_issues(issues, acc_issues) + do_fetch_issue_states_page(rest_ids, assignee_filter, graphql_fun, updated_acc, issue_order_index) + end {:error, reason} -> {:error, reason} end end + defp issue_order_index(ids) when is_list(ids) do + ids + |> Enum.with_index() + |> Map.new() + end + + defp sort_issues_by_requested_ids(issues, issue_order_index) + when is_list(issues) and is_map(issue_order_index) do + fallback_index = map_size(issue_order_index) + + Enum.sort_by(issues, fn + %Issue{id: issue_id} -> Map.get(issue_order_index, issue_id, fallback_index) + _ -> fallback_index + end) + end + defp build_graphql_payload(query, variables, operation_name) do %{ "query" => query, diff --git a/elixir/lib/symphony_elixir/orchestrator.ex b/elixir/lib/symphony_elixir/orchestrator.ex index 4286ce1f64..36ada1dcfa 100644 --- a/elixir/lib/symphony_elixir/orchestrator.ex +++ b/elixir/lib/symphony_elixir/orchestrator.ex @@ -31,6 +31,8 @@ defmodule SymphonyElixir.Orchestrator do :max_concurrent_agents, :next_poll_due_at_ms, :poll_check_in_progress, + :tick_timer_ref, + :tick_token, running: %{}, completed: MapSet.new(), claimed: MapSet.new(), @@ -56,20 +58,48 @@ defmodule SymphonyElixir.Orchestrator do max_concurrent_agents: config.agent.max_concurrent_agents, next_poll_due_at_ms: now_ms, poll_check_in_progress: false, + tick_timer_ref: nil, + tick_token: nil, codex_totals: @empty_codex_totals, codex_rate_limits: nil } run_terminal_workspace_cleanup() - :ok = schedule_tick(0) + state = schedule_tick(state, 0) {:ok, state} end @impl true + def handle_info({:tick, tick_token}, %{tick_token: tick_token} = state) + when is_reference(tick_token) do + state = refresh_runtime_config(state) + + state = %{ + state + | poll_check_in_progress: true, + next_poll_due_at_ms: nil, + tick_timer_ref: nil, + tick_token: nil + } + + notify_dashboard() + :ok = schedule_poll_cycle_start() + {:noreply, state} + end + + def handle_info({:tick, _tick_token}, state), do: {:noreply, state} + def handle_info(:tick, state) do state = refresh_runtime_config(state) - state = %{state | poll_check_in_progress: true, next_poll_due_at_ms: nil} + + state = %{ + state + | poll_check_in_progress: true, + next_poll_due_at_ms: nil, + tick_timer_ref: nil, + tick_token: nil + } notify_dashboard() :ok = schedule_poll_cycle_start() @@ -79,11 +109,8 @@ defmodule SymphonyElixir.Orchestrator do def handle_info(:run_poll_cycle, state) do state = refresh_runtime_config(state) state = maybe_dispatch(state) - now_ms = System.monotonic_time(:millisecond) - next_poll_due_at_ms = now_ms + state.poll_interval_ms - :ok = schedule_tick(state.poll_interval_ms) - - state = %{state | poll_check_in_progress: false, next_poll_due_at_ms: next_poll_due_at_ms} + state = schedule_tick(state, state.poll_interval_ms) + state = %{state | poll_check_in_progress: false} notify_dashboard() {:noreply, state} @@ -155,9 +182,9 @@ defmodule SymphonyElixir.Orchestrator do def handle_info({:codex_worker_update, _issue_id, _update}, state), do: {:noreply, state} - def handle_info({:retry_issue, issue_id}, state) do + def handle_info({:retry_issue, issue_id, retry_token}, state) do result = - case pop_retry_attempt_state(state, issue_id) do + case pop_retry_attempt_state(state, issue_id, retry_token) do {:ok, attempt, metadata, state} -> handle_retry_issue(state, issue_id, attempt, metadata) :missing -> {:noreply, state} end @@ -166,6 +193,8 @@ defmodule SymphonyElixir.Orchestrator do result end + def handle_info({:retry_issue, _issue_id}, state), do: {:noreply, state} + def handle_info(msg, state) do Logger.debug("Orchestrator ignored message: #{inspect(msg)}") {:noreply, state} @@ -231,12 +260,13 @@ defmodule SymphonyElixir.Orchestrator do else case Tracker.fetch_issue_states_by_ids(running_ids) do {:ok, issues} -> - reconcile_running_issue_states( - issues, + issues + |> reconcile_running_issue_states( state, active_state_set(), terminal_state_set() ) + |> reconcile_missing_running_issue_ids(running_ids, issues) {:error, reason} -> Logger.debug("Failed to refresh running issue states: #{inspect(reason)}; keeping active workers") @@ -311,6 +341,40 @@ defmodule SymphonyElixir.Orchestrator do defp reconcile_issue_state(_issue, state, _active_states, _terminal_states), do: state + defp reconcile_missing_running_issue_ids(%State{} = state, requested_issue_ids, issues) + when is_list(requested_issue_ids) and is_list(issues) do + visible_issue_ids = + issues + |> Enum.flat_map(fn + %Issue{id: issue_id} when is_binary(issue_id) -> [issue_id] + _ -> [] + end) + |> MapSet.new() + + Enum.reduce(requested_issue_ids, state, fn issue_id, state_acc -> + if MapSet.member?(visible_issue_ids, issue_id) do + state_acc + else + log_missing_running_issue(state_acc, issue_id) + terminate_running_issue(state_acc, issue_id, false) + end + end) + end + + defp reconcile_missing_running_issue_ids(state, _requested_issue_ids, _issues), do: state + + defp log_missing_running_issue(%State{} = state, issue_id) when is_binary(issue_id) do + case Map.get(state.running, issue_id) do + %{identifier: identifier} -> + Logger.info("Issue no longer visible during running-state refresh: issue_id=#{issue_id} issue_identifier=#{identifier}; stopping active agent") + + _ -> + Logger.info("Issue no longer visible during running-state refresh: issue_id=#{issue_id}; stopping active agent") + end + end + + defp log_missing_running_issue(_state, _issue_id), do: :ok + defp refresh_running_issue_state(%State{} = state, %Issue{} = issue) do case Map.get(state.running, issue.id) do %{issue: _} = running_entry -> @@ -669,6 +733,7 @@ defmodule SymphonyElixir.Orchestrator do next_attempt = if is_integer(attempt), do: attempt, else: previous_retry.attempt + 1 delay_ms = retry_delay(next_attempt, metadata) old_timer = Map.get(previous_retry, :timer_ref) + retry_token = make_ref() due_at_ms = System.monotonic_time(:millisecond) + delay_ms identifier = pick_retry_identifier(issue_id, previous_retry, metadata) error = pick_retry_error(previous_retry, metadata) @@ -677,7 +742,7 @@ defmodule SymphonyElixir.Orchestrator do Process.cancel_timer(old_timer) end - timer_ref = Process.send_after(self(), {:retry_issue, issue_id}, delay_ms) + timer_ref = Process.send_after(self(), {:retry_issue, issue_id, retry_token}, delay_ms) error_suffix = if is_binary(error), do: " error=#{error}", else: "" @@ -689,6 +754,7 @@ defmodule SymphonyElixir.Orchestrator do Map.put(state.retry_attempts, issue_id, %{ attempt: next_attempt, timer_ref: timer_ref, + retry_token: retry_token, due_at_ms: due_at_ms, identifier: identifier, error: error @@ -696,9 +762,9 @@ defmodule SymphonyElixir.Orchestrator do } end - defp pop_retry_attempt_state(%State{} = state, issue_id) do + defp pop_retry_attempt_state(%State{} = state, issue_id, retry_token) when is_reference(retry_token) do case Map.get(state.retry_attempts, issue_id) do - %{attempt: attempt} = retry_entry -> + %{attempt: attempt, retry_token: ^retry_token} = retry_entry -> metadata = %{ identifier: Map.get(retry_entry, :identifier), error: Map.get(retry_entry, :error) @@ -960,10 +1026,7 @@ defmodule SymphonyElixir.Orchestrator do now_ms = System.monotonic_time(:millisecond) already_due? = is_integer(state.next_poll_due_at_ms) and state.next_poll_due_at_ms <= now_ms coalesced = state.poll_check_in_progress == true or already_due? - - unless coalesced do - :ok = schedule_tick(0) - end + state = if coalesced, do: state, else: schedule_tick(state, 0) {:reply, %{ @@ -1048,9 +1111,20 @@ defmodule SymphonyElixir.Orchestrator do } end - defp schedule_tick(delay_ms) do - :timer.send_after(delay_ms, self(), :tick) - :ok + defp schedule_tick(%State{} = state, delay_ms) when is_integer(delay_ms) and delay_ms >= 0 do + if is_reference(state.tick_timer_ref) do + Process.cancel_timer(state.tick_timer_ref) + end + + tick_token = make_ref() + timer_ref = Process.send_after(self(), {:tick, tick_token}, delay_ms) + + %{ + state + | tick_timer_ref: timer_ref, + tick_token: tick_token, + next_poll_due_at_ms: System.monotonic_time(:millisecond) + delay_ms + } end defp schedule_poll_cycle_start do diff --git a/elixir/lib/symphony_elixir/path_safety.ex b/elixir/lib/symphony_elixir/path_safety.ex new file mode 100644 index 0000000000..fca59887ae --- /dev/null +++ b/elixir/lib/symphony_elixir/path_safety.ex @@ -0,0 +1,50 @@ +defmodule SymphonyElixir.PathSafety do + @moduledoc false + + @spec canonicalize(Path.t()) :: {:ok, Path.t()} | {:error, term()} + def canonicalize(path) when is_binary(path) do + expanded_path = Path.expand(path) + {root, segments} = split_absolute_path(expanded_path) + + case resolve_segments(root, [], segments) do + {:ok, canonical_path} -> + {:ok, canonical_path} + + {:error, reason} -> + {:error, {:path_canonicalize_failed, expanded_path, reason}} + end + end + + defp split_absolute_path(path) when is_binary(path) do + [root | segments] = Path.split(path) + {root, segments} + end + + defp resolve_segments(root, resolved_segments, []), do: {:ok, join_path(root, resolved_segments)} + + defp resolve_segments(root, resolved_segments, [segment | rest]) do + candidate_path = join_path(root, resolved_segments ++ [segment]) + + case File.lstat(candidate_path) do + {:ok, %File.Stat{type: :symlink}} -> + with {:ok, target} <- :file.read_link_all(String.to_charlist(candidate_path)) do + resolved_target = Path.expand(IO.chardata_to_string(target), join_path(root, resolved_segments)) + {target_root, target_segments} = split_absolute_path(resolved_target) + resolve_segments(target_root, [], target_segments ++ rest) + end + + {:ok, _stat} -> + resolve_segments(root, resolved_segments ++ [segment], rest) + + {:error, :enoent} -> + {:ok, join_path(root, resolved_segments ++ [segment | rest])} + + {:error, reason} -> + {:error, reason} + end + end + + defp join_path(root, segments) when is_list(segments) do + Enum.reduce(segments, root, fn segment, acc -> Path.join(acc, segment) end) + end +end diff --git a/elixir/lib/symphony_elixir/workspace.ex b/elixir/lib/symphony_elixir/workspace.ex index 2401573f7f..539976e93d 100644 --- a/elixir/lib/symphony_elixir/workspace.ex +++ b/elixir/lib/symphony_elixir/workspace.ex @@ -4,7 +4,7 @@ defmodule SymphonyElixir.Workspace do """ require Logger - alias SymphonyElixir.Config + alias SymphonyElixir.{Config, PathSafety} @excluded_entries MapSet.new([".elixir_ls", "tmp"]) @@ -15,9 +15,8 @@ defmodule SymphonyElixir.Workspace do try do safe_id = safe_identifier(issue_context.issue_identifier) - workspace = workspace_path_for_issue(safe_id) - - with :ok <- validate_workspace_path(workspace), + with {:ok, workspace} <- workspace_path_for_issue(safe_id), + :ok <- validate_workspace_path(workspace), {:ok, created?} <- ensure_workspace(workspace), :ok <- maybe_run_after_create_hook(workspace, issue_context, created?) do {:ok, workspace} @@ -71,9 +70,12 @@ defmodule SymphonyElixir.Workspace do @spec remove_issue_workspaces(term()) :: :ok def remove_issue_workspaces(identifier) when is_binary(identifier) do safe_id = safe_identifier(identifier) - workspace = Path.join(Config.settings!().workspace.root, safe_id) - remove(workspace) + case workspace_path_for_issue(safe_id) do + {:ok, workspace} -> remove(workspace) + {:error, _reason} -> :ok + end + :ok end @@ -111,7 +113,9 @@ defmodule SymphonyElixir.Workspace do end defp workspace_path_for_issue(safe_id) when is_binary(safe_id) do - Path.join(Config.settings!().workspace.root, safe_id) + Config.settings!().workspace.root + |> Path.join(safe_id) + |> PathSafety.canonicalize() end defp safe_identifier(identifier) do @@ -218,46 +222,29 @@ defmodule SymphonyElixir.Workspace do defp validate_workspace_path(workspace) when is_binary(workspace) do expanded_workspace = Path.expand(workspace) - root = Path.expand(Config.settings!().workspace.root) - root_prefix = root <> "/" + expanded_root = Path.expand(Config.settings!().workspace.root) + expanded_root_prefix = expanded_root <> "/" - cond do - expanded_workspace == root -> - {:error, {:workspace_equals_root, expanded_workspace, root}} + with {:ok, canonical_workspace} <- PathSafety.canonicalize(expanded_workspace), + {:ok, canonical_root} <- PathSafety.canonicalize(expanded_root) do + canonical_root_prefix = canonical_root <> "/" - String.starts_with?(expanded_workspace <> "/", root_prefix) -> - ensure_no_symlink_components(expanded_workspace, root) + cond do + canonical_workspace == canonical_root -> + {:error, {:workspace_equals_root, canonical_workspace, canonical_root}} - true -> - {:error, {:workspace_outside_root, expanded_workspace, root}} - end - end - - defp ensure_no_symlink_components(workspace, root) do - workspace - |> Path.relative_to(root) - |> Path.split() - |> Enum.reduce_while(root, fn segment, current_path -> - next_path = Path.join(current_path, segment) - - case File.lstat(next_path) do - {:ok, %File.Stat{type: :symlink}} -> - {:halt, {:error, {:workspace_symlink_escape, next_path, root}}} + String.starts_with?(canonical_workspace <> "/", canonical_root_prefix) -> + :ok - {:ok, _stat} -> - {:cont, next_path} + String.starts_with?(expanded_workspace <> "/", expanded_root_prefix) -> + {:error, {:workspace_symlink_escape, expanded_workspace, canonical_root}} - {:error, :enoent} -> - {:halt, :ok} - - {:error, reason} -> - {:halt, {:error, {:workspace_path_unreadable, next_path, reason}}} + true -> + {:error, {:workspace_outside_root, canonical_workspace, canonical_root}} end - end) - |> case do - :ok -> :ok - {:error, _reason} = error -> error - _final_path -> :ok + else + {:error, {:path_canonicalize_failed, path, reason}} -> + {:error, {:workspace_path_unreadable, path, reason}} end end diff --git a/elixir/test/symphony_elixir/app_server_test.exs b/elixir/test/symphony_elixir/app_server_test.exs index 20ab61e9c1..3b98c44343 100644 --- a/elixir/test/symphony_elixir/app_server_test.exs +++ b/elixir/test/symphony_elixir/app_server_test.exs @@ -39,6 +39,150 @@ defmodule SymphonyElixir.AppServerTest do end end + test "app server rejects symlink escape cwd paths under the workspace root" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-app-server-symlink-cwd-guard-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + outside_workspace = Path.join(test_root, "outside") + symlink_workspace = Path.join(workspace_root, "MT-1000") + + File.mkdir_p!(workspace_root) + File.mkdir_p!(outside_workspace) + File.ln_s!(outside_workspace, symlink_workspace) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root + ) + + issue = %Issue{ + id: "issue-workspace-symlink-guard", + identifier: "MT-1000", + title: "Validate symlink workspace guard", + description: "Ensure app-server refuses symlink escape cwd targets", + state: "In Progress", + url: "https://example.org/issues/MT-1000", + labels: ["backend"] + } + + assert {:error, {:invalid_workspace_cwd, :symlink_escape, ^symlink_workspace, _root}} = + AppServer.run(symlink_workspace, "guard", issue) + after + File.rm_rf(test_root) + end + end + + test "app server passes explicit turn sandbox policies through unchanged" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-app-server-supported-turn-policies-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + workspace = Path.join(workspace_root, "MT-1001") + codex_binary = Path.join(test_root, "fake-codex") + trace_file = Path.join(test_root, "codex-supported-turn-policies.trace") + previous_trace = System.get_env("SYMP_TEST_CODEx_TRACE") + + on_exit(fn -> + if is_binary(previous_trace) do + System.put_env("SYMP_TEST_CODEx_TRACE", previous_trace) + else + System.delete_env("SYMP_TEST_CODEx_TRACE") + end + end) + + System.put_env("SYMP_TEST_CODEx_TRACE", trace_file) + File.mkdir_p!(workspace) + + File.write!(codex_binary, """ + #!/bin/sh + trace_file="${SYMP_TEST_CODEx_TRACE:-/tmp/codex-supported-turn-policies.trace}" + count=0 + + while IFS= read -r line; do + count=$((count + 1)) + printf 'JSON:%s\\n' "$line" >> "$trace_file" + + case "$count" in + 1) + printf '%s\\n' '{"id":1,"result":{}}' + ;; + 2) + printf '%s\\n' '{"id":2,"result":{"thread":{"id":"thread-1001"}}}' + ;; + 3) + printf '%s\\n' '{"id":3,"result":{"turn":{"id":"turn-1001"}}}' + ;; + 4) + printf '%s\\n' '{"method":"turn/completed"}' + exit 0 + ;; + *) + exit 0 + ;; + esac + done + """) + + File.chmod!(codex_binary, 0o755) + + issue = %Issue{ + id: "issue-supported-turn-policies", + identifier: "MT-1001", + title: "Validate explicit turn sandbox policy passthrough", + description: "Ensure runtime startup forwards configured turn sandbox policies unchanged", + state: "In Progress", + url: "https://example.org/issues/MT-1001", + labels: ["backend"] + } + + policy_cases = [ + %{"type" => "dangerFullAccess"}, + %{"type" => "externalSandbox", "profile" => "remote-ci"}, + %{"type" => "workspaceWrite", "writableRoots" => ["relative/path"], "networkAccess" => true}, + %{"type" => "futureSandbox", "nested" => %{"flag" => true}} + ] + + Enum.each(policy_cases, fn configured_policy -> + File.rm(trace_file) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + codex_command: "#{codex_binary} app-server", + codex_turn_sandbox_policy: configured_policy + ) + + assert {:ok, _result} = AppServer.run(workspace, "Validate supported turn policy", issue) + + trace = File.read!(trace_file) + lines = String.split(trace, "\n", trim: true) + + assert Enum.any?(lines, fn line -> + if String.starts_with?(line, "JSON:") do + line + |> String.trim_leading("JSON:") + |> Jason.decode!() + |> then(fn payload -> + payload["method"] == "turn/start" && + get_in(payload, ["params", "sandboxPolicy"]) == configured_policy + end) + else + false + end + end) + end) + after + File.rm_rf(test_root) + end + end + test "app server marks request-for-input events as a hard failure" do test_root = Path.join( diff --git a/elixir/test/symphony_elixir/core_test.exs b/elixir/test/symphony_elixir/core_test.exs index 023305eb72..fa96b7b091 100644 --- a/elixir/test/symphony_elixir/core_test.exs +++ b/elixir/test/symphony_elixir/core_test.exs @@ -350,6 +350,84 @@ defmodule SymphonyElixir.CoreTest do end end + test "missing running issues stop active agents without cleaning the workspace" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-missing-running-reconcile-#{System.unique_integer([:positive])}" + ) + + previous_memory_issues = Application.get_env(:symphony_elixir, :memory_tracker_issues) + issue_id = "issue-missing" + issue_identifier = "MT-557" + + try do + write_workflow_file!(Workflow.workflow_file_path(), + tracker_kind: "memory", + workspace_root: test_root, + tracker_active_states: ["Todo", "In Progress", "In Review"], + tracker_terminal_states: ["Closed", "Cancelled", "Canceled", "Duplicate"], + poll_interval_ms: 30_000 + ) + + Application.put_env(:symphony_elixir, :memory_tracker_issues, []) + + orchestrator_name = Module.concat(__MODULE__, :MissingRunningIssueOrchestrator) + {:ok, pid} = Orchestrator.start_link(name: orchestrator_name) + + on_exit(fn -> + restore_app_env(:memory_tracker_issues, previous_memory_issues) + + if Process.alive?(pid) do + Process.exit(pid, :normal) + end + end) + + Process.sleep(50) + + assert {:ok, workspace} = + SymphonyElixir.PathSafety.canonicalize(Path.join(test_root, issue_identifier)) + + File.mkdir_p!(workspace) + + agent_pid = + spawn(fn -> + receive do + :stop -> :ok + end + end) + + initial_state = :sys.get_state(pid) + + running_entry = %{ + pid: agent_pid, + ref: nil, + identifier: issue_identifier, + issue: %Issue{id: issue_id, state: "In Progress", identifier: issue_identifier}, + started_at: DateTime.utc_now() + } + + :sys.replace_state(pid, fn _ -> + initial_state + |> Map.put(:running, %{issue_id => running_entry}) + |> Map.put(:claimed, MapSet.new([issue_id])) + |> Map.put(:retry_attempts, %{}) + end) + + send(pid, :tick) + Process.sleep(100) + state = :sys.get_state(pid) + + refute Map.has_key?(state.running, issue_id) + refute MapSet.member?(state.claimed, issue_id) + refute Process.alive?(agent_pid) + assert File.exists?(workspace) + after + restore_app_env(:memory_tracker_issues, previous_memory_issues) + File.rm_rf(test_root) + end + end + test "reconcile updates running issue state for active issues" do issue_id = "issue-3" @@ -555,6 +633,76 @@ defmodule SymphonyElixir.CoreTest do assert_due_in_range(due_at_ms, 9_000, 10_500) end + test "stale retry timer messages do not consume newer retry entries" do + issue_id = "issue-stale-retry" + orchestrator_name = Module.concat(__MODULE__, :StaleRetryOrchestrator) + {:ok, pid} = Orchestrator.start_link(name: orchestrator_name) + + on_exit(fn -> + if Process.alive?(pid) do + Process.exit(pid, :normal) + end + end) + + initial_state = :sys.get_state(pid) + current_retry_token = make_ref() + stale_retry_token = make_ref() + + :sys.replace_state(pid, fn _ -> + initial_state + |> Map.put(:retry_attempts, %{ + issue_id => %{ + attempt: 2, + timer_ref: nil, + retry_token: current_retry_token, + due_at_ms: System.monotonic_time(:millisecond) + 30_000, + identifier: "MT-561", + error: "agent exited: :boom" + } + }) + end) + + send(pid, {:retry_issue, issue_id, stale_retry_token}) + Process.sleep(50) + + assert %{ + attempt: 2, + retry_token: ^current_retry_token, + identifier: "MT-561", + error: "agent exited: :boom" + } = :sys.get_state(pid).retry_attempts[issue_id] + end + + test "manual refresh coalesces repeated requests and ignores superseded ticks" do + now_ms = System.monotonic_time(:millisecond) + stale_tick_token = make_ref() + + state = %Orchestrator.State{ + poll_interval_ms: 30_000, + max_concurrent_agents: 1, + next_poll_due_at_ms: now_ms + 30_000, + poll_check_in_progress: false, + tick_timer_ref: nil, + tick_token: stale_tick_token, + codex_totals: %{input_tokens: 0, output_tokens: 0, total_tokens: 0, seconds_running: 0}, + codex_rate_limits: nil + } + + assert {:reply, %{queued: true, coalesced: false}, refreshed_state} = + Orchestrator.handle_call(:request_refresh, {self(), make_ref()}, state) + + assert is_reference(refreshed_state.tick_timer_ref) + assert is_reference(refreshed_state.tick_token) + refute refreshed_state.tick_token == stale_tick_token + assert refreshed_state.next_poll_due_at_ms <= System.monotonic_time(:millisecond) + + assert {:reply, %{queued: true, coalesced: true}, coalesced_state} = + Orchestrator.handle_call(:request_refresh, {self(), make_ref()}, refreshed_state) + + assert coalesced_state.tick_token == refreshed_state.tick_token + assert {:noreply, ^coalesced_state} = Orchestrator.handle_info({:tick, stale_tick_token}, coalesced_state) + end + defp assert_due_in_range(due_at_ms, min_remaining_ms, max_remaining_ms) do remaining_ms = due_at_ms - System.monotonic_time(:millisecond) @@ -562,6 +710,9 @@ defmodule SymphonyElixir.CoreTest do assert remaining_ms <= max_remaining_ms end + defp restore_app_env(key, nil), do: Application.delete_env(:symphony_elixir, key) + defp restore_app_env(key, value), do: Application.put_env(:symphony_elixir, key, value) + test "fetch issues by states with empty state set is a no-op" do assert {:ok, []} = Client.fetch_issues_by_states([]) end @@ -1268,6 +1419,7 @@ defmodule SymphonyElixir.CoreTest do } assert {:ok, _result} = AppServer.run(workspace, "Fix workspace start args", issue) + assert {:ok, canonical_workspace} = SymphonyElixir.PathSafety.canonicalize(workspace) trace = File.read!(trace_file) lines = String.split(trace, "\n", trim: true) @@ -1295,7 +1447,7 @@ defmodule SymphonyElixir.CoreTest do payload["method"] == "thread/start" && get_in(payload, ["params", "approvalPolicy"]) == expected_approval_policy && get_in(payload, ["params", "sandbox"]) == "workspace-write" && - get_in(payload, ["params", "cwd"]) == Path.expand(workspace) + get_in(payload, ["params", "cwd"]) == canonical_workspace end) else false @@ -1304,7 +1456,7 @@ defmodule SymphonyElixir.CoreTest do expected_turn_sandbox_policy = %{ "type" => "workspaceWrite", - "writableRoots" => [Path.expand(workspace)], + "writableRoots" => [canonical_workspace], "readOnlyAccess" => %{"type" => "fullAccess"}, "networkAccess" => false, "excludeTmpdirEnvVar" => false, @@ -1326,7 +1478,7 @@ defmodule SymphonyElixir.CoreTest do } payload["method"] == "turn/start" && - get_in(payload, ["params", "cwd"]) == Path.expand(workspace) && + get_in(payload, ["params", "cwd"]) == canonical_workspace && get_in(payload, ["params", "approvalPolicy"]) == expected_approval_policy && get_in(payload, ["params", "sandboxPolicy"]) == expected_turn_sandbox_policy end) @@ -1481,6 +1633,9 @@ defmodule SymphonyElixir.CoreTest do File.chmod!(codex_binary, 0o755) + workspace_cache = Path.join(Path.expand(workspace), ".cache") + File.mkdir_p!(workspace_cache) + write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root, codex_command: "#{codex_binary} app-server", @@ -1488,7 +1643,7 @@ defmodule SymphonyElixir.CoreTest do codex_thread_sandbox: "workspace-write", codex_turn_sandbox_policy: %{ type: "workspaceWrite", - writableRoots: [Path.expand(workspace), Path.join(Path.expand(workspace_root), ".cache")] + writableRoots: [Path.expand(workspace), workspace_cache] } ) @@ -1523,7 +1678,7 @@ defmodule SymphonyElixir.CoreTest do expected_turn_policy = %{ "type" => "workspaceWrite", - "writableRoots" => [Path.expand(workspace), Path.join(Path.expand(workspace_root), ".cache")] + "writableRoots" => [Path.expand(workspace), workspace_cache] } assert Enum.any?(lines, fn line -> diff --git a/elixir/test/symphony_elixir/workspace_and_config_test.exs b/elixir/test/symphony_elixir/workspace_and_config_test.exs index 7b681d54f5..03a3c59ee4 100644 --- a/elixir/test/symphony_elixir/workspace_and_config_test.exs +++ b/elixir/test/symphony_elixir/workspace_and_config_test.exs @@ -106,8 +106,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root) + assert {:ok, canonical_workspace} = SymphonyElixir.PathSafety.canonicalize(stale_workspace) assert {:ok, workspace} = Workspace.create_for_issue("MT-STALE") - assert workspace == stale_workspace + assert workspace == canonical_workspace assert File.dir?(workspace) after File.rm_rf(workspace_root) @@ -132,13 +133,43 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root) - assert {:error, {:workspace_symlink_escape, ^symlink_path, ^workspace_root}} = + assert {:ok, canonical_outside_root} = SymphonyElixir.PathSafety.canonicalize(outside_root) + assert {:ok, canonical_workspace_root} = SymphonyElixir.PathSafety.canonicalize(workspace_root) + + assert {:error, {:workspace_outside_root, ^canonical_outside_root, ^canonical_workspace_root}} = Workspace.create_for_issue("MT-SYM") after File.rm_rf(test_root) end end + test "workspace canonicalizes symlinked workspace roots before creating issue directories" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-workspace-root-symlink-#{System.unique_integer([:positive])}" + ) + + try do + actual_root = Path.join(test_root, "actual-workspaces") + linked_root = Path.join(test_root, "linked-workspaces") + + File.mkdir_p!(actual_root) + File.ln_s!(actual_root, linked_root) + + write_workflow_file!(Workflow.workflow_file_path(), workspace_root: linked_root) + + assert {:ok, canonical_workspace} = + SymphonyElixir.PathSafety.canonicalize(Path.join(actual_root, "MT-LINK")) + + assert {:ok, workspace} = Workspace.create_for_issue("MT-LINK") + assert workspace == canonical_workspace + assert File.dir?(workspace) + after + File.rm_rf(test_root) + end + end + test "workspace remove rejects the workspace root itself with a distinct error" do workspace_root = Path.join( @@ -150,7 +181,10 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do File.mkdir_p!(workspace_root) write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root) - assert {:error, {:workspace_equals_root, ^workspace_root, ^workspace_root}, ""} = + assert {:ok, canonical_workspace_root} = + SymphonyElixir.PathSafety.canonicalize(workspace_root) + + assert {:error, {:workspace_equals_root, ^canonical_workspace_root, ^canonical_workspace_root}, ""} = Workspace.remove(workspace_root) after File.rm_rf(workspace_root) @@ -209,8 +243,9 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root) workspace = Path.join(workspace_root, "MT-608") + assert {:ok, canonical_workspace} = SymphonyElixir.PathSafety.canonicalize(workspace) - assert {:ok, ^workspace} = Workspace.create_for_issue("MT-608") + assert {:ok, ^canonical_workspace} = Workspace.create_for_issue("MT-608") assert File.dir?(workspace) assert {:ok, []} = File.ls(workspace) after @@ -351,6 +386,49 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert Enum.map(merged, & &1.identifier) == ["MT-1", "MT-2", "MT-3"] end + test "linear client paginates issue state fetches by id beyond one page" do + issue_ids = Enum.map(1..55, &"issue-#{&1}") + first_batch_ids = Enum.take(issue_ids, 50) + second_batch_ids = Enum.drop(issue_ids, 50) + + raw_issue = fn issue_id -> + suffix = String.replace_prefix(issue_id, "issue-", "") + + %{ + "id" => issue_id, + "identifier" => "MT-#{suffix}", + "title" => "Issue #{suffix}", + "description" => "Description #{suffix}", + "state" => %{"name" => "In Progress"}, + "labels" => %{"nodes" => []}, + "inverseRelations" => %{"nodes" => []} + } + end + + graphql_fun = fn query, variables -> + send(self(), {:fetch_issue_states_page, query, variables}) + + body = %{ + "data" => %{ + "issues" => %{ + "nodes" => Enum.map(variables.ids, raw_issue) + } + } + } + + {:ok, body} + end + + assert {:ok, issues} = Client.fetch_issue_states_by_ids_for_test(issue_ids, graphql_fun) + + assert Enum.map(issues, & &1.id) == issue_ids + + assert_receive {:fetch_issue_states_page, query, %{ids: ^first_batch_ids, first: 50, relationFirst: 50}} + assert query =~ "SymphonyLinearIssuesById" + + assert_receive {:fetch_issue_states_page, ^query, %{ids: ^second_batch_ids, first: 5, relationFirst: 50}} + end + test "linear client logs response bodies for non-200 graphql responses" do log = ExUnit.CaptureLog.capture_log(fn -> @@ -677,9 +755,12 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert config.codex.thread_sandbox == "workspace-write" + assert {:ok, canonical_default_workspace_root} = + SymphonyElixir.PathSafety.canonicalize(Path.join(System.tmp_dir!(), "symphony_workspaces")) + assert Config.codex_turn_sandbox_policy() == %{ "type" => "workspaceWrite", - "writableRoots" => [Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces"))], + "writableRoots" => [canonical_default_workspace_root], "readOnlyAccess" => %{"type" => "fullAccess"}, "networkAccess" => false, "excludeTmpdirEnvVar" => false, @@ -693,19 +774,35 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do write_workflow_file!(Workflow.workflow_file_path(), codex_command: "codex app-server --model gpt-5.3-codex") assert Config.settings!().codex.command == "codex app-server --model gpt-5.3-codex" + explicit_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-explicit-sandbox-root-#{System.unique_integer([:positive])}" + ) + + explicit_workspace = Path.join(explicit_root, "MT-EXPLICIT") + explicit_cache = Path.join(explicit_workspace, "cache") + File.mkdir_p!(explicit_cache) + + on_exit(fn -> File.rm_rf(explicit_root) end) + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: explicit_root, codex_approval_policy: "on-request", codex_thread_sandbox: "workspace-write", - codex_turn_sandbox_policy: %{type: "workspaceWrite", writableRoots: ["/tmp/workspace", "/tmp/cache"]} + codex_turn_sandbox_policy: %{ + type: "workspaceWrite", + writableRoots: [explicit_workspace, explicit_cache] + } ) config = Config.settings!() assert config.codex.approval_policy == "on-request" assert config.codex.thread_sandbox == "workspace-write" - assert Config.codex_turn_sandbox_policy() == %{ + assert Config.codex_turn_sandbox_policy(explicit_workspace) == %{ "type" => "workspaceWrite", - "writableRoots" => ["/tmp/workspace", "/tmp/cache"] + "writableRoots" => [explicit_workspace, explicit_cache] } write_workflow_file!(Workflow.workflow_file_path(), tracker_active_states: ",") @@ -770,13 +867,13 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert config.codex.approval_policy == "future-policy" assert config.codex.thread_sandbox == "future-sandbox" + assert :ok = Config.validate!() + assert Config.codex_turn_sandbox_policy() == %{ "type" => "futureSandbox", "nested" => %{"flag" => true} } - assert :ok = Config.validate!() - write_workflow_file!(Workflow.workflow_file_path(), codex_command: "codex app-server") assert Config.settings!().codex.command == "codex app-server" end @@ -976,6 +1073,110 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do } end + test "runtime sandbox policy resolution passes explicit policies through unchanged" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-runtime-sandbox-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + issue_workspace = Path.join(workspace_root, "MT-100") + File.mkdir_p!(issue_workspace) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + codex_turn_sandbox_policy: %{ + type: "workspaceWrite", + writableRoots: ["relative/path"], + networkAccess: true + } + ) + + assert {:ok, runtime_settings} = Config.codex_runtime_settings(issue_workspace) + + assert runtime_settings.turn_sandbox_policy == %{ + "type" => "workspaceWrite", + "writableRoots" => ["relative/path"], + "networkAccess" => true + } + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + codex_turn_sandbox_policy: %{ + type: "futureSandbox", + nested: %{flag: true} + } + ) + + assert {:ok, runtime_settings} = Config.codex_runtime_settings(issue_workspace) + + assert runtime_settings.turn_sandbox_policy == %{ + "type" => "futureSandbox", + "nested" => %{"flag" => true} + } + after + File.rm_rf(test_root) + end + end + + test "path safety returns errors for invalid path segments" do + invalid_segment = String.duplicate("a", 300) + path = Path.join(System.tmp_dir!(), invalid_segment) + expanded_path = Path.expand(path) + + assert {:error, {:path_canonicalize_failed, ^expanded_path, :enametoolong}} = + SymphonyElixir.PathSafety.canonicalize(path) + end + + test "runtime sandbox policy resolution defaults when omitted and ignores workspace for explicit policies" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-runtime-sandbox-branches-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + issue_workspace = Path.join(workspace_root, "MT-101") + + File.mkdir_p!(issue_workspace) + + write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root) + + settings = Config.settings!() + + assert {:ok, canonical_workspace_root} = + SymphonyElixir.PathSafety.canonicalize(workspace_root) + + assert {:ok, default_policy} = Schema.resolve_runtime_turn_sandbox_policy(settings) + assert default_policy["type"] == "workspaceWrite" + assert default_policy["writableRoots"] == [canonical_workspace_root] + + read_only_settings = %{ + settings + | codex: %{settings.codex | turn_sandbox_policy: %{"type" => "readOnly", "networkAccess" => true}} + } + + assert {:ok, %{"type" => "readOnly", "networkAccess" => true}} = + Schema.resolve_runtime_turn_sandbox_policy(read_only_settings, 123) + + future_settings = %{ + settings + | codex: %{settings.codex | turn_sandbox_policy: %{"type" => "futureSandbox", "nested" => %{"flag" => true}}} + } + + assert {:ok, %{"type" => "futureSandbox", "nested" => %{"flag" => true}}} = + Schema.resolve_runtime_turn_sandbox_policy(future_settings, 123) + + assert {:error, {:unsafe_turn_sandbox_policy, {:invalid_workspace_root, 123}}} = + Schema.resolve_runtime_turn_sandbox_policy(settings, 123) + after + File.rm_rf(test_root) + end + end + test "workflow prompt is used when building base prompt" do workflow_prompt = "Workflow prompt body used as codex instruction." From b1863e83ac9197fefa110e7102ac03f000fd29f4 Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Tue, 10 Mar 2026 13:45:29 -0700 Subject: [PATCH 04/41] Add a real live Linear/Codex E2E target #### Context Add a real end-to-end path for Symphony Elixir that exercises Linear and Codex together, and make it easy to invoke explicitly from one Make target. #### TL;DR *Add a real live Linear/Codex E2E test plus a `make e2e` entrypoint.* #### Summary - Add an opt-in live Elixir test that creates a real Linear project and issue. - Run a real Codex turn and verify both workspace output and Linear side effects. - Require Codex to post a Linear comment and close the issue before completion. - Add `make e2e` with clear failures for missing `LINEAR_API_KEY` or `codex`. - Document the new E2E entrypoint and behavior in the Elixir README. #### Alternatives - Keep using ad hoc `mix test` commands and environment toggles. - Keep shimming Linear or post-processing issue state locally. - Those keep the happy path less reproducible and do not prove Codex can mutate Linear itself during the turn. #### Test Plan - [x] `make -C elixir all` - [x] `env -u LINEAR_API_KEY make -C elixir e2e` - [x] `LINEAR_API_KEY=$(tr -d '\r\n' < ~/.linear_api_key) SYMPHONY_RUN_LIVE_E2E=1 mix test test/symphony_elixir/live_e2e_test.exs` --- elixir/Makefile | 17 +- elixir/README.md | 19 + elixir/test/symphony_elixir/live_e2e_test.exs | 460 ++++++++++++++++++ 3 files changed, 494 insertions(+), 2 deletions(-) create mode 100644 elixir/test/symphony_elixir/live_e2e_test.exs diff --git a/elixir/Makefile b/elixir/Makefile index 9c1ae9096d..331eaef518 100644 --- a/elixir/Makefile +++ b/elixir/Makefile @@ -1,9 +1,9 @@ -.PHONY: help all setup deps build fmt fmt-check lint test coverage ci dialyzer +.PHONY: help all setup deps build fmt fmt-check lint test coverage ci dialyzer e2e MIX ?= mix help: - @echo "Targets: setup, deps, fmt, fmt-check, lint, test, coverage, dialyzer, ci" + @echo "Targets: setup, deps, fmt, fmt-check, lint, test, coverage, dialyzer, e2e, ci" setup: $(MIX) setup @@ -33,6 +33,19 @@ dialyzer: $(MIX) deps.get $(MIX) dialyzer --format short +e2e: + @if [ -z "$$LINEAR_API_KEY" ]; then \ + echo "LINEAR_API_KEY is required for \`make e2e\`."; \ + echo "Export it first, for example:"; \ + echo " export LINEAR_API_KEY=\$$(tr -d '\\r\\n' < ~/.linear_api_key)"; \ + exit 1; \ + fi + @if ! command -v codex >/dev/null 2>&1; then \ + echo "\`codex\` must be on PATH for \`make e2e\`."; \ + exit 1; \ + fi + SYMPHONY_RUN_LIVE_E2E=1 $(MIX) test test/symphony_elixir/live_e2e_test.exs + ci: $(MAKE) setup $(MAKE) build diff --git a/elixir/README.md b/elixir/README.md index 004b5943a4..7c93acf07a 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -173,6 +173,25 @@ The observability UI now runs on a minimal Phoenix stack: make all ``` +Run the real external end-to-end test only when you want Symphony to create disposable Linear +resources and launch a real `codex app-server` session: + +```bash +cd elixir +export LINEAR_API_KEY=... +make e2e +``` + +Optional environment variables: + +- `SYMPHONY_LIVE_LINEAR_TEAM_KEY` defaults to `SYME2E` +- `SYMPHONY_LIVE_CODEX_COMMAND` defaults to `codex app-server` + +The live test creates a temporary Linear project and issue, writes a temporary `WORKFLOW.md`, +runs a real agent turn, verifies the workspace side effect, requires Codex to comment on and close +the Linear issue, then marks the project completed so the run remains visible in Linear. +`make e2e` fails fast with a clear error if `LINEAR_API_KEY` is unset. + ## FAQ ### Why Elixir? diff --git a/elixir/test/symphony_elixir/live_e2e_test.exs b/elixir/test/symphony_elixir/live_e2e_test.exs new file mode 100644 index 0000000000..b775e66fcc --- /dev/null +++ b/elixir/test/symphony_elixir/live_e2e_test.exs @@ -0,0 +1,460 @@ +defmodule SymphonyElixir.LiveE2ETest do + use SymphonyElixir.TestSupport + + require Logger + + @moduletag :live_e2e + @moduletag timeout: 300_000 + + @default_team_key "SYME2E" + @result_file "LIVE_E2E_RESULT.txt" + @live_e2e_skip_reason (cond do + System.get_env("SYMPHONY_RUN_LIVE_E2E") != "1" -> + "set SYMPHONY_RUN_LIVE_E2E=1 to enable the real Linear/Codex end-to-end test" + + is_nil(System.find_executable("codex")) -> + "real Codex live test requires `codex` on PATH" + + System.get_env("LINEAR_API_KEY") in [nil, ""] -> + "real Linear live test requires LINEAR_API_KEY" + + true -> + nil + end) + + @team_query """ + query SymphonyLiveE2ETeam($key: String!) { + teams(filter: {key: {eq: $key}}, first: 1) { + nodes { + id + key + name + states(first: 50) { + nodes { + id + name + type + } + } + } + } + } + """ + + @create_project_mutation """ + mutation SymphonyLiveE2ECreateProject($name: String!, $teamIds: [String!]!) { + projectCreate(input: {name: $name, teamIds: $teamIds}) { + success + project { + id + name + slugId + url + } + } + } + """ + + @create_issue_mutation """ + mutation SymphonyLiveE2ECreateIssue( + $teamId: String! + $projectId: String! + $title: String! + $description: String! + $stateId: String + ) { + issueCreate( + input: { + teamId: $teamId + projectId: $projectId + title: $title + description: $description + stateId: $stateId + } + ) { + success + issue { + id + identifier + title + description + url + state { + name + } + } + } + } + """ + + @project_statuses_query """ + query SymphonyLiveE2EProjectStatuses { + projectStatuses(first: 50) { + nodes { + id + name + type + } + } + } + """ + + @issue_details_query """ + query SymphonyLiveE2EIssueDetails($id: String!) { + issue(id: $id) { + id + identifier + state { + name + type + } + comments(first: 20) { + nodes { + body + } + } + } + } + """ + + @complete_project_mutation """ + mutation SymphonyLiveE2ECompleteProject($id: String!, $statusId: String!, $completedAt: DateTime!) { + projectUpdate(id: $id, input: {statusId: $statusId, completedAt: $completedAt}) { + success + } + } + """ + + @tag skip: @live_e2e_skip_reason + test "creates a real Linear project and issue, then runs a real Codex turn" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-live-e2e-#{System.unique_integer([:positive])}" + ) + + workflow_root = Path.join(test_root, "workflow") + workflow_file = Path.join(workflow_root, "WORKFLOW.md") + workspace_root = Path.join(test_root, "workspaces") + team_key = System.get_env("SYMPHONY_LIVE_LINEAR_TEAM_KEY") || @default_team_key + codex_command = System.get_env("SYMPHONY_LIVE_CODEX_COMMAND") || "codex app-server" + original_workflow_path = Workflow.workflow_file_path() + + File.mkdir_p!(workflow_root) + + try do + Workflow.set_workflow_file_path(workflow_file) + + write_workflow_file!(workflow_file, + tracker_api_token: "$LINEAR_API_KEY", + tracker_project_slug: "bootstrap", + workspace_root: workspace_root, + codex_command: codex_command, + codex_approval_policy: "never", + observability_enabled: false + ) + + team = fetch_team!(team_key) + active_state = active_state!(team) + completed_project_status = completed_project_status!() + terminal_states = terminal_state_names(team) + + project = + create_project!( + team["id"], + "Symphony Live E2E #{System.unique_integer([:positive])}" + ) + + issue = + create_issue!( + team["id"], + project["id"], + active_state["id"], + "Symphony live e2e issue for #{project["name"]}" + ) + + write_workflow_file!(workflow_file, + tracker_api_token: "$LINEAR_API_KEY", + tracker_project_slug: project["slugId"], + tracker_active_states: [active_state["name"]], + tracker_terminal_states: terminal_states, + workspace_root: workspace_root, + codex_command: codex_command, + codex_approval_policy: "never", + codex_turn_timeout_ms: 600_000, + codex_stall_timeout_ms: 600_000, + observability_enabled: false, + prompt: live_prompt(project["slugId"]) + ) + + assert :ok = AgentRunner.run(issue, nil, max_turns: 1) + + result_path = Path.join([workspace_root, issue.identifier, @result_file]) + assert File.exists?(result_path) + assert File.read!(result_path) == expected_result(issue.identifier, project["slugId"]) + + issue_snapshot = fetch_issue_details!(issue.id) + assert issue_completed?(issue_snapshot) + assert issue_has_comment?(issue_snapshot, expected_comment(issue.identifier, project["slugId"])) + + assert :ok = complete_project(project["id"], completed_project_status["id"]) + after + Workflow.set_workflow_file_path(original_workflow_path) + File.rm_rf(test_root) + end + end + + defp fetch_team!(team_key) do + @team_query + |> graphql_data!(%{key: team_key}) + |> get_in(["teams", "nodes"]) + |> case do + [team | _] -> + team + + _ -> + flunk("expected Linear team #{inspect(team_key)} to exist") + end + end + + defp active_state!(%{"states" => %{"nodes" => states}}) when is_list(states) do + Enum.find(states, &(&1["type"] == "started")) || + Enum.find(states, &(&1["type"] == "unstarted")) || + Enum.find(states, &(&1["type"] not in ["completed", "canceled"])) || + flunk("expected team to expose at least one non-terminal workflow state") + end + + defp terminal_state_names(%{"states" => %{"nodes" => states}}) when is_list(states) do + states + |> Enum.filter(&(&1["type"] in ["completed", "canceled"])) + |> Enum.map(& &1["name"]) + |> case do + [] -> ["Done", "Canceled", "Cancelled"] + names -> names + end + end + + defp completed_project_status! do + @project_statuses_query + |> graphql_data!(%{}) + |> get_in(["projectStatuses", "nodes"]) + |> case do + statuses when is_list(statuses) -> + Enum.find(statuses, &(&1["type"] == "completed")) || + flunk("expected workspace to expose a completed project status") + + payload -> + flunk("expected project statuses list, got: #{inspect(payload)}") + end + end + + defp create_project!(team_id, name) do + @create_project_mutation + |> graphql_data!(%{teamIds: [team_id], name: name}) + |> fetch_successful_entity!("projectCreate", "project") + end + + defp create_issue!(team_id, project_id, state_id, title) do + issue = + @create_issue_mutation + |> graphql_data!(%{ + teamId: team_id, + projectId: project_id, + title: title, + description: title, + stateId: state_id + }) + |> fetch_successful_entity!("issueCreate", "issue") + + %Issue{ + id: issue["id"], + identifier: issue["identifier"], + title: issue["title"], + description: issue["description"], + state: get_in(issue, ["state", "name"]), + url: issue["url"], + labels: [], + blocked_by: [] + } + end + + defp complete_project(project_id, completed_status_id) + when is_binary(project_id) and is_binary(completed_status_id) do + update_entity( + @complete_project_mutation, + %{ + id: project_id, + statusId: completed_status_id, + completedAt: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() + }, + "projectUpdate", + "project" + ) + end + + defp fetch_issue_details!(issue_id) when is_binary(issue_id) do + @issue_details_query + |> graphql_data!(%{id: issue_id}) + |> get_in(["issue"]) + |> case do + %{} = issue -> issue + payload -> flunk("expected issue details payload, got: #{inspect(payload)}") + end + end + + defp issue_completed?(%{"state" => %{"type" => type}}), do: type in ["completed", "canceled"] + defp issue_completed?(_issue), do: false + + defp issue_has_comment?(%{"comments" => %{"nodes" => comments}}, expected_body) when is_list(comments) do + Enum.any?(comments, &(&1["body"] == expected_body)) + end + + defp issue_has_comment?(_issue, _expected_body), do: false + + defp update_entity(mutation, variables, mutation_name, entity_name) do + case Client.graphql(mutation, variables) do + {:ok, %{"data" => %{^mutation_name => %{"success" => true}}}} -> + :ok + + {:ok, %{"errors" => errors}} -> + Logger.warning("Live e2e finalization failed for #{entity_name}: #{inspect(errors)}") + :ok + + {:ok, payload} -> + Logger.warning("Live e2e finalization failed for #{entity_name}: #{inspect(payload)}") + :ok + + {:error, reason} -> + Logger.warning("Live e2e finalization failed for #{entity_name}: #{inspect(reason)}") + :ok + end + end + + defp graphql_data!(query, variables) when is_binary(query) and is_map(variables) do + case Client.graphql(query, variables) do + {:ok, %{"data" => data, "errors" => errors}} when is_map(data) and is_list(errors) -> + flunk("Linear GraphQL returned partial errors: #{inspect(errors)}") + + {:ok, %{"errors" => errors}} when is_list(errors) -> + flunk("Linear GraphQL failed: #{inspect(errors)}") + + {:ok, %{"data" => data}} when is_map(data) -> + data + + {:ok, payload} -> + flunk("Linear GraphQL returned unexpected payload: #{inspect(payload)}") + + {:error, reason} -> + flunk("Linear GraphQL request failed: #{inspect(reason)}") + end + end + + defp fetch_successful_entity!(data, mutation_name, entity_name) + when is_map(data) and is_binary(mutation_name) and is_binary(entity_name) do + case data do + %{^mutation_name => %{"success" => true, ^entity_name => %{} = entity}} -> + entity + + _ -> + flunk("expected successful #{mutation_name} response, got: #{inspect(data)}") + end + end + + defp live_prompt(project_slug) do + """ + You are running a real Symphony end-to-end test. + + The current working directory is the workspace root. + + Step 1: + Create a file named #{@result_file} in the current working directory by running exactly: + + ```sh + cat > #{@result_file} <<'EOF' + identifier={{ issue.identifier }} + project_slug=#{project_slug} + EOF + ``` + + Then verify it by running: + + ```sh + cat #{@result_file} + ``` + + The file content must be exactly: + identifier={{ issue.identifier }} + project_slug=#{project_slug} + + Step 2: + Use the `linear_graphql` tool to query the current issue by `{{ issue.id }}` and read: + - existing comments + - team workflow states + + If the exact comment body below is not already present, post exactly one comment on the current issue with this exact body: + #{expected_comment("{{ issue.identifier }}", project_slug)} + + Use these exact GraphQL operations: + + ```graphql + query IssueContext($id: String!) { + issue(id: $id) { + comments(first: 20) { + nodes { + body + } + } + team { + states(first: 50) { + nodes { + id + name + type + } + } + } + } + } + ``` + + ```graphql + mutation AddComment($issueId: String!, $body: String!) { + commentCreate(input: {issueId: $issueId, body: $body}) { + success + } + } + ``` + + Step 3: + Use the same issue-context query result to choose a workflow state whose `type` is `completed`. + Then move the current issue to that state with this exact mutation: + + ```graphql + mutation CompleteIssue($id: String!, $stateId: String!) { + issueUpdate(id: $id, input: {stateId: $stateId}) { + success + } + } + ``` + + Step 4: + Verify all outcomes with one final `linear_graphql` query against `{{ issue.id }}`: + - the exact comment body is present + - the issue state type is `completed` + + Do not ask for approval. + Stop only after all three conditions are true: + 1. the file exists with the exact contents above + 2. the Linear comment exists with the exact body above + 3. the Linear issue is in a completed terminal state + """ + end + + defp expected_result(issue_identifier, project_slug) do + "identifier=#{issue_identifier}\nproject_slug=#{project_slug}\n" + end + + defp expected_comment(issue_identifier, project_slug) do + "Symphony live e2e comment\nidentifier=#{issue_identifier}\nproject_slug=#{project_slug}" + end +end From ff65c7c729c03d4daa550bd30290fc5291f60c67 Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Wed, 11 Mar 2026 14:47:47 -0700 Subject: [PATCH 05/41] Add SSH worker support to Symphony Elixir #### Context Symphony needs to run tickets on SSH workers, keep the live E2E reliable, and preserve correct remote path semantics. #### TL;DR *Add SSH workers, Docker-backed SSH live E2E coverage, per-host caps, and correct worker-side `~` resolution.* #### Summary - Add SSH worker execution, remote workspace handling, and SSH app-server launch support in Elixir. - Add live E2E coverage for both local workers and SSH workers, with Docker-backed SSH workers by default. - Add an optional shared `worker.max_concurrent_agents_per_host` cap and document the SSH extension. - Keep `workspace.root` raw so SSH workers resolve `~` on the worker, while local paths expand at local use sites. #### Alternatives - Keep expanding `workspace.root` in config and carry duplicate raw path state, but that complicates the config model. - Require only absolute remote workspace roots, but that makes real worker setup less ergonomic and less representative. #### Test Plan - [x] `make -C elixir all` - [x] `cd elixir && env -u SYMPHONY_LIVE_SSH_WORKER_HOSTS LINEAR_API_KEY="$(tr -d '\r\n' < ~/.linear_api_key)" SYMPHONY_RUN_LIVE_E2E=1 mix test test/symphony_elixir/live_e2e_test.exs:128` --- SPEC.md | 65 +++ elixir/Makefile | 10 - elixir/README.md | 21 +- elixir/lib/symphony_elixir/agent_runner.ex | 102 +++- .../lib/symphony_elixir/codex/app_server.ex | 132 ++++- .../lib/symphony_elixir/codex/dynamic_tool.ex | 19 +- elixir/lib/symphony_elixir/config.ex | 7 +- elixir/lib/symphony_elixir/config/schema.ex | 78 ++- elixir/lib/symphony_elixir/orchestrator.ex | 170 +++++- elixir/lib/symphony_elixir/ssh.ex | 100 ++++ elixir/lib/symphony_elixir/workspace.ex | 290 ++++++++-- elixir/lib/symphony_elixir_web/presenter.ex | 25 +- .../test/support/live_e2e_docker/Dockerfile | 22 + .../live_e2e_docker/docker-compose.yml | 20 + .../live_e2e_docker/live_worker_entrypoint.sh | 13 + .../live_e2e_docker/symphony-live-worker.conf | 7 + elixir/test/support/test_support.exs | 20 + .../test/symphony_elixir/app_server_test.exs | 137 ++++- elixir/test/symphony_elixir/core_test.exs | 47 ++ .../symphony_elixir/dynamic_tool_test.exs | 108 +--- .../test/symphony_elixir/extensions_test.exs | 13 +- elixir/test/symphony_elixir/live_e2e_test.exs | 520 +++++++++++++++--- .../orchestrator_status_test.exs | 2 + elixir/test/symphony_elixir/ssh_test.exs | 199 +++++++ .../workspace_and_config_test.exs | 124 ++++- 25 files changed, 1917 insertions(+), 334 deletions(-) create mode 100644 elixir/lib/symphony_elixir/ssh.ex create mode 100644 elixir/test/support/live_e2e_docker/Dockerfile create mode 100644 elixir/test/support/live_e2e_docker/docker-compose.yml create mode 100644 elixir/test/support/live_e2e_docker/live_worker_entrypoint.sh create mode 100644 elixir/test/support/live_e2e_docker/symphony-live-worker.conf create mode 100644 elixir/test/symphony_elixir/ssh_test.exs diff --git a/SPEC.md b/SPEC.md index e861f3b9e6..f9e2b63a14 100644 --- a/SPEC.md +++ b/SPEC.md @@ -559,6 +559,10 @@ This section is intentionally redundant so a coding agent can implement the conf - `tracker.terminal_states`: list of strings, default `["Closed", "Cancelled", "Canceled", "Duplicate", "Done"]` - `polling.interval_ms`: integer, default `30000` - `workspace.root`: path, default `/symphony_workspaces` +- `worker.ssh_hosts` (extension): list of SSH host strings, optional; when omitted, work runs + locally +- `worker.max_concurrent_agents_per_host` (extension): positive integer, optional; shared per-host + cap applied across configured SSH hosts - `hooks.after_create`: shell script or null - `hooks.before_run`: shell script or null - `hooks.after_run`: shell script or null @@ -729,6 +733,12 @@ Per-state limit: The runtime counts issues by their current tracked state in the `running` map. +Optional SSH host limit: + +- When `worker.max_concurrent_agents_per_host` is set, each configured SSH host may run at most + that many concurrent agents at once. +- Hosts at that cap are skipped for new dispatch until capacity frees up. + ### 8.4 Retry and Backoff Retry entry creation: @@ -2108,3 +2118,58 @@ Use the same validation profiles as Section 17: - Verify hook execution and workflow path resolution on the target host OS/shell environment. - If the optional HTTP server is shipped, verify the configured port behavior and loopback/default bind expectations on the target environment. + +## Appendix A. SSH Worker Extension (Optional) + +This appendix describes a common extension profile in which Symphony keeps one central +orchestrator but executes worker runs on one or more remote hosts over SSH. + +### A.1 Execution Model + +- The orchestrator remains the single source of truth for polling, claims, retries, and + reconciliation. +- `worker.ssh_hosts` provides the candidate SSH destinations for remote execution. +- Each worker run is assigned to one host at a time, and that host becomes part of the run's + effective execution identity along with the issue workspace. +- `workspace.root` is interpreted on the remote host, not on the orchestrator host. +- The coding-agent app-server is launched over SSH stdio instead of as a local subprocess, so the + orchestrator still owns the session lifecycle even though commands execute remotely. +- Continuation turns inside one worker lifetime should stay on the same host and workspace. +- A remote host should satisfy the same basic contract as a local worker environment: reachable + shell, writable workspace root, coding-agent executable, and any required auth or repository + prerequisites. + +### A.2 Scheduling Notes + +- SSH hosts may be treated as a pool for dispatch. +- Implementations may prefer the previously used host on retries when that host is still + available. +- `worker.max_concurrent_agents_per_host` is an optional shared per-host cap across configured SSH + hosts. +- When all SSH hosts are at capacity, dispatch should wait rather than silently falling back to a + different execution mode. +- Implementations may fail over to another host when the original host is unavailable before work + has meaningfully started. +- Once a run has already produced side effects, a transparent rerun on another host should be + treated as a new attempt, not as invisible failover. + +### A.3 Problems to Consider + +- Remote environment drift: + - Each host needs the expected shell environment, coding-agent executable, auth, and repository + prerequisites. +- Workspace locality: + - Workspaces are usually host-local, so moving an issue to a different host is typically a cold + restart unless shared storage exists. +- Path and command safety: + - Remote path resolution, shell quoting, and workspace-boundary checks matter more once execution + crosses a machine boundary. +- Startup and failover semantics: + - Implementations should distinguish host-connectivity/startup failures from in-workspace agent + failures so the same ticket is not accidentally re-executed on multiple hosts. +- Host health and saturation: + - A dead or overloaded host should reduce available capacity, not cause duplicate execution or an + accidental fallback to local work. +- Cleanup and observability: + - Operators need to know which host owns a run, where its workspace lives, and whether cleanup + happened on the right machine. diff --git a/elixir/Makefile b/elixir/Makefile index 331eaef518..61c40270aa 100644 --- a/elixir/Makefile +++ b/elixir/Makefile @@ -34,16 +34,6 @@ dialyzer: $(MIX) dialyzer --format short e2e: - @if [ -z "$$LINEAR_API_KEY" ]; then \ - echo "LINEAR_API_KEY is required for \`make e2e\`."; \ - echo "Export it first, for example:"; \ - echo " export LINEAR_API_KEY=\$$(tr -d '\\r\\n' < ~/.linear_api_key)"; \ - exit 1; \ - fi - @if ! command -v codex >/dev/null 2>&1; then \ - echo "\`codex\` must be on PATH for \`make e2e\`."; \ - exit 1; \ - fi SYMPHONY_RUN_LIVE_E2E=1 $(MIX) test test/symphony_elixir/live_e2e_test.exs ci: diff --git a/elixir/README.md b/elixir/README.md index 7c93acf07a..603b4bb000 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -185,12 +185,23 @@ make e2e Optional environment variables: - `SYMPHONY_LIVE_LINEAR_TEAM_KEY` defaults to `SYME2E` -- `SYMPHONY_LIVE_CODEX_COMMAND` defaults to `codex app-server` +- `SYMPHONY_LIVE_SSH_WORKER_HOSTS` uses those SSH hosts when set, as a comma-separated list -The live test creates a temporary Linear project and issue, writes a temporary `WORKFLOW.md`, -runs a real agent turn, verifies the workspace side effect, requires Codex to comment on and close -the Linear issue, then marks the project completed so the run remains visible in Linear. -`make e2e` fails fast with a clear error if `LINEAR_API_KEY` is unset. +`make e2e` runs two live scenarios: +- one with a local worker +- one with SSH workers + +If `SYMPHONY_LIVE_SSH_WORKER_HOSTS` is unset, the SSH scenario uses `docker compose` to start two +disposable SSH workers on `localhost:`. The live test generates a temporary SSH keypair, +mounts the host `~/.codex/auth.json` into each worker, verifies that Symphony can talk to them +over real SSH, then runs the same orchestration flow against those worker addresses. This keeps +the transport representative without depending on long-lived external machines. + +Set `SYMPHONY_LIVE_SSH_WORKER_HOSTS` if you want `make e2e` to target real SSH hosts instead. + +The live test creates a temporary Linear project and issue, writes a temporary `WORKFLOW.md`, runs +a real agent turn, verifies the workspace side effect, requires Codex to comment on and close the +Linear issue, then marks the project completed so the run remains visible in Linear. ## FAQ diff --git a/elixir/lib/symphony_elixir/agent_runner.ex b/elixir/lib/symphony_elixir/agent_runner.ex index 482d547685..35ea8a03e9 100644 --- a/elixir/lib/symphony_elixir/agent_runner.ex +++ b/elixir/lib/symphony_elixir/agent_runner.ex @@ -7,28 +7,58 @@ defmodule SymphonyElixir.AgentRunner do alias SymphonyElixir.Codex.AppServer alias SymphonyElixir.{Config, Linear.Issue, PromptBuilder, Tracker, Workspace} + @type worker_host :: String.t() | nil + @spec run(map(), pid() | nil, keyword()) :: :ok | no_return() def run(issue, codex_update_recipient \\ nil, opts \\ []) do - Logger.info("Starting agent run for #{issue_context(issue)}") + worker_hosts = + candidate_worker_hosts(Keyword.get(opts, :worker_host), Config.settings!().worker.ssh_hosts) + + Logger.info("Starting agent run for #{issue_context(issue)} worker_hosts=#{inspect(worker_hosts_for_log(worker_hosts))}") + + case run_on_worker_hosts(issue, codex_update_recipient, opts, worker_hosts) do + :ok -> + :ok + + {:error, reason} -> + Logger.error("Agent run failed for #{issue_context(issue)}: #{inspect(reason)}") + raise RuntimeError, "Agent run failed for #{issue_context(issue)}: #{inspect(reason)}" + end + end - case Workspace.create_for_issue(issue) do + defp run_on_worker_hosts(issue, codex_update_recipient, opts, [worker_host | rest]) do + case run_on_worker_host(issue, codex_update_recipient, opts, worker_host) do + :ok -> + :ok + + {:error, reason} when rest != [] -> + Logger.warning("Agent run failed for #{issue_context(issue)} worker_host=#{worker_host_for_log(worker_host)} reason=#{inspect(reason)}; trying next worker host") + run_on_worker_hosts(issue, codex_update_recipient, opts, rest) + + {:error, reason} -> + {:error, reason} + end + end + + defp run_on_worker_hosts(_issue, _codex_update_recipient, _opts, []), do: {:error, :no_worker_hosts_available} + + defp run_on_worker_host(issue, codex_update_recipient, opts, worker_host) do + Logger.info("Starting worker attempt for #{issue_context(issue)} worker_host=#{worker_host_for_log(worker_host)}") + + case Workspace.create_for_issue(issue, worker_host) do {:ok, workspace} -> + send_worker_runtime_info(codex_update_recipient, issue, worker_host, workspace) + try do - with :ok <- Workspace.run_before_run_hook(workspace, issue), - :ok <- run_codex_turns(workspace, issue, codex_update_recipient, opts) do - :ok - else - {:error, reason} -> - Logger.error("Agent run failed for #{issue_context(issue)}: #{inspect(reason)}") - raise RuntimeError, "Agent run failed for #{issue_context(issue)}: #{inspect(reason)}" + with :ok <- Workspace.run_before_run_hook(workspace, issue, worker_host) do + run_codex_turns(workspace, issue, codex_update_recipient, opts, worker_host) end after - Workspace.run_after_run_hook(workspace, issue) + Workspace.run_after_run_hook(workspace, issue, worker_host) end {:error, reason} -> - Logger.error("Agent run failed for #{issue_context(issue)}: #{inspect(reason)}") - raise RuntimeError, "Agent run failed for #{issue_context(issue)}: #{inspect(reason)}" + {:error, reason} end end @@ -46,11 +76,27 @@ defmodule SymphonyElixir.AgentRunner do defp send_codex_update(_recipient, _issue, _message), do: :ok - defp run_codex_turns(workspace, issue, codex_update_recipient, opts) do + defp send_worker_runtime_info(recipient, %Issue{id: issue_id}, worker_host, workspace) + when is_binary(issue_id) and is_pid(recipient) and is_binary(workspace) do + send( + recipient, + {:worker_runtime_info, issue_id, + %{ + worker_host: worker_host, + workspace_path: workspace + }} + ) + + :ok + end + + defp send_worker_runtime_info(_recipient, _issue, _worker_host, _workspace), do: :ok + + defp run_codex_turns(workspace, issue, codex_update_recipient, opts, worker_host) do max_turns = Keyword.get(opts, :max_turns, Config.settings!().agent.max_turns) issue_state_fetcher = Keyword.get(opts, :issue_state_fetcher, &Tracker.fetch_issue_states_by_ids/1) - with {:ok, session} <- AppServer.start_session(workspace) do + with {:ok, session} <- AppServer.start_session(workspace, worker_host: worker_host) do try do do_run_codex_turns(session, workspace, issue, codex_update_recipient, opts, issue_state_fetcher, 1, max_turns) after @@ -142,6 +188,34 @@ defmodule SymphonyElixir.AgentRunner do defp active_issue_state?(_state_name), do: false + defp candidate_worker_hosts(nil, []), do: [nil] + + defp candidate_worker_hosts(preferred_host, configured_hosts) when is_list(configured_hosts) do + hosts = + configured_hosts + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> Enum.uniq() + + case preferred_host do + host when is_binary(host) and host != "" -> + [host | Enum.reject(hosts, &(&1 == host))] + + _ when hosts == [] -> + [nil] + + _ -> + hosts + end + end + + defp worker_hosts_for_log(worker_hosts) do + Enum.map(worker_hosts, &worker_host_for_log/1) + end + + defp worker_host_for_log(nil), do: "local" + defp worker_host_for_log(worker_host), do: worker_host + defp normalize_issue_state(state_name) when is_binary(state_name) do state_name |> String.trim() diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index ef79022417..7da87ce998 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -4,7 +4,7 @@ defmodule SymphonyElixir.Codex.AppServer do """ require Logger - alias SymphonyElixir.{Codex.DynamicTool, Config, PathSafety} + alias SymphonyElixir.{Codex.DynamicTool, Config, PathSafety, SSH} @initialize_id 1 @thread_start_id 2 @@ -21,12 +21,13 @@ defmodule SymphonyElixir.Codex.AppServer do thread_sandbox: String.t(), turn_sandbox_policy: map(), thread_id: String.t(), - workspace: Path.t() + workspace: Path.t(), + worker_host: String.t() | nil } @spec run(Path.t(), String.t(), map(), keyword()) :: {:ok, map()} | {:error, term()} def run(workspace, prompt, issue, opts \\ []) do - with {:ok, session} <- start_session(workspace) do + with {:ok, session} <- start_session(workspace, opts) do try do run_turn(session, prompt, issue, opts) after @@ -35,13 +36,15 @@ defmodule SymphonyElixir.Codex.AppServer do end end - @spec start_session(Path.t()) :: {:ok, session()} | {:error, term()} - def start_session(workspace) do - with {:ok, expanded_workspace} <- validate_workspace_cwd(workspace), - {:ok, port} <- start_port(expanded_workspace) do - metadata = port_metadata(port) + @spec start_session(Path.t(), keyword()) :: {:ok, session()} | {:error, term()} + def start_session(workspace, opts \\ []) do + worker_host = Keyword.get(opts, :worker_host) - with {:ok, session_policies} <- session_policies(expanded_workspace), + with {:ok, expanded_workspace} <- validate_workspace_cwd(workspace, worker_host), + {:ok, port} <- start_port(expanded_workspace, worker_host) do + metadata = port_metadata(port, worker_host) + + with {:ok, session_policies} <- session_policies(expanded_workspace, worker_host), {:ok, thread_id} <- do_start_session(port, expanded_workspace, session_policies) do {:ok, %{ @@ -52,7 +55,8 @@ defmodule SymphonyElixir.Codex.AppServer do thread_sandbox: session_policies.thread_sandbox, turn_sandbox_policy: session_policies.turn_sandbox_policy, thread_id: thread_id, - workspace: expanded_workspace + workspace: expanded_workspace, + worker_host: worker_host }} else {:error, reason} -> @@ -140,7 +144,7 @@ defmodule SymphonyElixir.Codex.AppServer do stop_port(port) end - defp validate_workspace_cwd(workspace) when is_binary(workspace) do + defp validate_workspace_cwd(workspace, nil) when is_binary(workspace) do expanded_workspace = Path.expand(workspace) expanded_root = Path.expand(Config.settings!().workspace.root) expanded_root_prefix = expanded_root <> "/" @@ -168,7 +172,21 @@ defmodule SymphonyElixir.Codex.AppServer do end end - defp start_port(workspace) do + defp validate_workspace_cwd(workspace, worker_host) + when is_binary(workspace) and is_binary(worker_host) do + cond do + String.trim(workspace) == "" -> + {:error, {:invalid_workspace_cwd, :empty_remote_workspace, worker_host}} + + String.contains?(workspace, ["\n", "\r", <<0>>]) -> + {:error, {:invalid_workspace_cwd, :invalid_remote_workspace, worker_host, workspace}} + + true -> + {:ok, workspace} + end + end + + defp start_port(workspace, nil) do executable = System.find_executable("bash") if is_nil(executable) do @@ -191,13 +209,32 @@ defmodule SymphonyElixir.Codex.AppServer do end end - defp port_metadata(port) when is_port(port) do - case :erlang.port_info(port, :os_pid) do - {:os_pid, os_pid} -> - %{codex_app_server_pid: to_string(os_pid)} + defp start_port(workspace, worker_host) when is_binary(worker_host) do + remote_command = remote_launch_command(workspace) + SSH.start_port(worker_host, remote_command, line: @port_line_bytes) + end - _ -> - %{} + defp remote_launch_command(workspace) when is_binary(workspace) do + [ + "cd #{shell_escape(workspace)}", + "exec #{Config.settings!().codex.command}" + ] + |> Enum.join(" && ") + end + + defp port_metadata(port, worker_host) when is_port(port) do + base_metadata = + case :erlang.port_info(port, :os_pid) do + {:os_pid, os_pid} -> + %{codex_app_server_pid: to_string(os_pid)} + + _ -> + %{} + end + + case worker_host do + host when is_binary(host) -> Map.put(base_metadata, :worker_host, host) + _ -> base_metadata end end @@ -225,10 +262,14 @@ defmodule SymphonyElixir.Codex.AppServer do end end - defp session_policies(workspace) do + defp session_policies(workspace, nil) do Config.codex_runtime_settings(workspace) end + defp session_policies(workspace, worker_host) when is_binary(worker_host) do + Config.codex_runtime_settings(workspace, remote: true) + end + defp do_start_session(port, workspace, session_policies) do case send_initialize(port) do :ok -> start_thread(port, workspace, session_policies) @@ -243,7 +284,7 @@ defmodule SymphonyElixir.Codex.AppServer do "params" => %{ "approvalPolicy" => approval_policy, "sandbox" => thread_sandbox, - "cwd" => Path.expand(workspace), + "cwd" => workspace, "dynamicTools" => DynamicTool.tool_specs() } }) @@ -272,7 +313,7 @@ defmodule SymphonyElixir.Codex.AppServer do "text" => prompt } ], - "cwd" => Path.expand(workspace), + "cwd" => workspace, "title" => "#{issue.identifier}: #{issue.title}", "approvalPolicy" => approval_policy, "sandboxPolicy" => turn_sandbox_policy @@ -515,7 +556,10 @@ defmodule SymphonyElixir.Codex.AppServer do tool_name = tool_call_name(params) arguments = tool_call_arguments(params) - result = tool_executor.(tool_name, arguments) + result = + tool_name + |> tool_executor.(arguments) + |> normalize_dynamic_tool_result() send_message(port, %{ "id" => id, @@ -635,6 +679,44 @@ defmodule SymphonyElixir.Codex.AppServer do :unhandled end + defp normalize_dynamic_tool_result(%{"success" => success} = result) when is_boolean(success) do + output = + case Map.get(result, "output") do + existing_output when is_binary(existing_output) -> existing_output + _ -> dynamic_tool_output(result) + end + + content_items = + case Map.get(result, "contentItems") do + existing_items when is_list(existing_items) -> existing_items + _ -> dynamic_tool_content_items(output) + end + + result + |> Map.put("output", output) + |> Map.put("contentItems", content_items) + end + + defp normalize_dynamic_tool_result(result) do + %{ + "success" => false, + "output" => inspect(result), + "contentItems" => dynamic_tool_content_items(inspect(result)) + } + end + + defp dynamic_tool_output(%{"contentItems" => [%{"text" => text} | _]}) when is_binary(text), do: text + defp dynamic_tool_output(result), do: Jason.encode!(result, pretty: true) + + defp dynamic_tool_content_items(output) when is_binary(output) do + [ + %{ + "type" => "inputText", + "text" => output + } + ] + end + defp approve_or_require( port, id, @@ -921,7 +1003,7 @@ defmodule SymphonyElixir.Codex.AppServer do end defp metadata_from_message(port, payload) do - port |> port_metadata() |> maybe_set_usage(payload) + port |> port_metadata(nil) |> maybe_set_usage(payload) end defp maybe_set_usage(metadata, payload) when is_map(payload) do @@ -936,6 +1018,10 @@ defmodule SymphonyElixir.Codex.AppServer do defp maybe_set_usage(metadata, _payload), do: metadata + defp shell_escape(value) when is_binary(value) do + "'" <> String.replace(value, "'", "'\"'\"'") <> "'" + end + defp default_on_message(_message), do: :ok defp tool_call_name(params) when is_map(params) do diff --git a/elixir/lib/symphony_elixir/codex/dynamic_tool.ex b/elixir/lib/symphony_elixir/codex/dynamic_tool.ex index 716d360705..446c7fd2c8 100644 --- a/elixir/lib/symphony_elixir/codex/dynamic_tool.ex +++ b/elixir/lib/symphony_elixir/codex/dynamic_tool.ex @@ -118,24 +118,21 @@ defmodule SymphonyElixir.Codex.DynamicTool do _ -> true end - %{ - "success" => success, - "contentItems" => [ - %{ - "type" => "inputText", - "text" => encode_payload(response) - } - ] - } + dynamic_tool_response(success, encode_payload(response)) end defp failure_response(payload) do + dynamic_tool_response(false, encode_payload(payload)) + end + + defp dynamic_tool_response(success, output) when is_boolean(success) and is_binary(output) do %{ - "success" => false, + "success" => success, + "output" => output, "contentItems" => [ %{ "type" => "inputText", - "text" => encode_payload(payload) + "text" => output } ] } diff --git a/elixir/lib/symphony_elixir/config.ex b/elixir/lib/symphony_elixir/config.ex index 87011ac134..00e7f9b7ef 100644 --- a/elixir/lib/symphony_elixir/config.ex +++ b/elixir/lib/symphony_elixir/config.ex @@ -98,11 +98,12 @@ defmodule SymphonyElixir.Config do end end - @spec codex_runtime_settings(Path.t() | nil) :: {:ok, codex_runtime_settings()} | {:error, term()} - def codex_runtime_settings(workspace \\ nil) do + @spec codex_runtime_settings(Path.t() | nil, keyword()) :: + {:ok, codex_runtime_settings()} | {:error, term()} + def codex_runtime_settings(workspace \\ nil, opts \\ []) do with {:ok, settings} <- settings() do with {:ok, turn_sandbox_policy} <- - Schema.resolve_runtime_turn_sandbox_policy(settings, workspace) do + Schema.resolve_runtime_turn_sandbox_policy(settings, workspace, opts) do {:ok, %{ approval_policy: settings.codex.approval_policy, diff --git a/elixir/lib/symphony_elixir/config/schema.ex b/elixir/lib/symphony_elixir/config/schema.ex index c9ad948f97..17ead4ad6b 100644 --- a/elixir/lib/symphony_elixir/config/schema.ex +++ b/elixir/lib/symphony_elixir/config/schema.ex @@ -100,6 +100,25 @@ defmodule SymphonyElixir.Config.Schema do end end + defmodule Worker do + @moduledoc false + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + embedded_schema do + field(:ssh_hosts, {:array, :string}, default: []) + field(:max_concurrent_agents_per_host, :integer) + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(schema, attrs) do + schema + |> cast(attrs, [:ssh_hosts, :max_concurrent_agents_per_host], empty_values: []) + |> validate_number(:max_concurrent_agents_per_host, greater_than: 0) + end + end + defmodule Agent do @moduledoc false use Ecto.Schema @@ -246,6 +265,7 @@ defmodule SymphonyElixir.Config.Schema do embeds_one(:tracker, Tracker, on_replace: :update, defaults_to_struct: true) embeds_one(:polling, Polling, on_replace: :update, defaults_to_struct: true) embeds_one(:workspace, Workspace, on_replace: :update, defaults_to_struct: true) + embeds_one(:worker, Worker, on_replace: :update, defaults_to_struct: true) embeds_one(:agent, Agent, on_replace: :update, defaults_to_struct: true) embeds_one(:codex, Codex, on_replace: :update, defaults_to_struct: true) embeds_one(:hooks, Hooks, on_replace: :update, defaults_to_struct: true) @@ -276,19 +296,24 @@ defmodule SymphonyElixir.Config.Schema do policy _ -> - default_turn_sandbox_policy(workspace || settings.workspace.root) + workspace + |> default_workspace_root(settings.workspace.root) + |> expand_local_workspace_root() + |> default_turn_sandbox_policy() end end - @spec resolve_runtime_turn_sandbox_policy(%__MODULE__{}, Path.t() | nil) :: + @spec resolve_runtime_turn_sandbox_policy(%__MODULE__{}, Path.t() | nil, keyword()) :: {:ok, map()} | {:error, term()} - def resolve_runtime_turn_sandbox_policy(settings, workspace \\ nil) do + def resolve_runtime_turn_sandbox_policy(settings, workspace \\ nil, opts \\ []) do case settings.codex.turn_sandbox_policy do %{} = policy -> {:ok, policy} _ -> - default_runtime_turn_sandbox_policy(workspace || settings.workspace.root) + workspace + |> default_workspace_root(settings.workspace.root) + |> default_runtime_turn_sandbox_policy(opts) end end @@ -332,6 +357,7 @@ defmodule SymphonyElixir.Config.Schema do |> cast_embed(:tracker, with: &Tracker.changeset/2) |> cast_embed(:polling, with: &Polling.changeset/2) |> cast_embed(:workspace, with: &Workspace.changeset/2) + |> cast_embed(:worker, with: &Worker.changeset/2) |> cast_embed(:agent, with: &Agent.changeset/2) |> cast_embed(:codex, with: &Codex.changeset/2) |> cast_embed(:hooks, with: &Hooks.changeset/2) @@ -399,13 +425,13 @@ defmodule SymphonyElixir.Config.Schema do defp resolve_path_value(value, default) when is_binary(value) do case normalize_path_token(value) do :missing -> - Path.expand(default) + default "" -> - Path.expand(default) + default path -> - Path.expand(path) + path end end @@ -454,16 +480,9 @@ defmodule SymphonyElixir.Config.Schema do defp normalize_secret_value(_value), do: nil defp default_turn_sandbox_policy(workspace) do - writable_root = - if is_binary(workspace) and workspace != "" do - Path.expand(workspace) - else - Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) - end - %{ "type" => "workspaceWrite", - "writableRoots" => [writable_root], + "writableRoots" => [workspace], "readOnlyAccess" => %{"type" => "fullAccess"}, "networkAccess" => false, "excludeTmpdirEnvVar" => false, @@ -471,16 +490,37 @@ defmodule SymphonyElixir.Config.Schema do } end - defp default_runtime_turn_sandbox_policy(workspace_root) when is_binary(workspace_root) do - with {:ok, canonical_workspace_root} <- PathSafety.canonicalize(workspace_root) do - {:ok, default_turn_sandbox_policy(canonical_workspace_root)} + defp default_runtime_turn_sandbox_policy(workspace_root, opts) when is_binary(workspace_root) do + if Keyword.get(opts, :remote, false) do + {:ok, default_turn_sandbox_policy(workspace_root)} + else + with expanded_workspace_root <- expand_local_workspace_root(workspace_root), + {:ok, canonical_workspace_root} <- PathSafety.canonicalize(expanded_workspace_root) do + {:ok, default_turn_sandbox_policy(canonical_workspace_root)} + end end end - defp default_runtime_turn_sandbox_policy(workspace_root) do + defp default_runtime_turn_sandbox_policy(workspace_root, _opts) do {:error, {:unsafe_turn_sandbox_policy, {:invalid_workspace_root, workspace_root}}} end + defp default_workspace_root(workspace, _fallback) when is_binary(workspace) and workspace != "", + do: workspace + + defp default_workspace_root(nil, fallback), do: fallback + defp default_workspace_root("", fallback), do: fallback + defp default_workspace_root(workspace, _fallback), do: workspace + + defp expand_local_workspace_root(workspace_root) + when is_binary(workspace_root) and workspace_root != "" do + Path.expand(workspace_root) + end + + defp expand_local_workspace_root(_workspace_root) do + Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + end + defp format_errors(changeset) do changeset |> traverse_errors(&translate_error/1) diff --git a/elixir/lib/symphony_elixir/orchestrator.ex b/elixir/lib/symphony_elixir/orchestrator.ex index 36ada1dcfa..3cd814829b 100644 --- a/elixir/lib/symphony_elixir/orchestrator.ex +++ b/elixir/lib/symphony_elixir/orchestrator.ex @@ -138,7 +138,9 @@ defmodule SymphonyElixir.Orchestrator do |> complete_issue(issue_id) |> schedule_issue_retry(issue_id, 1, %{ identifier: running_entry.identifier, - delay_type: :continuation + delay_type: :continuation, + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path) }) _ -> @@ -148,7 +150,9 @@ defmodule SymphonyElixir.Orchestrator do schedule_issue_retry(state, issue_id, next_attempt, %{ identifier: running_entry.identifier, - error: "agent exited: #{inspect(reason)}" + error: "agent exited: #{inspect(reason)}", + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path) }) end @@ -159,6 +163,23 @@ defmodule SymphonyElixir.Orchestrator do end end + def handle_info({:worker_runtime_info, issue_id, runtime_info}, %{running: running} = state) + when is_binary(issue_id) and is_map(runtime_info) do + case Map.get(running, issue_id) do + nil -> + {:noreply, state} + + running_entry -> + updated_running_entry = + running_entry + |> maybe_put_runtime_value(:worker_host, runtime_info[:worker_host]) + |> maybe_put_runtime_value(:workspace_path, runtime_info[:workspace_path]) + + notify_dashboard() + {:noreply, %{state | running: Map.put(running, issue_id, updated_running_entry)}} + end + end + def handle_info( {:codex_worker_update, issue_id, %{event: _, timestamp: _} = update}, %{running: running} = state @@ -306,6 +327,12 @@ defmodule SymphonyElixir.Orchestrator do sort_issues_for_dispatch(issues) end + @doc false + @spec select_worker_host_for_test(term(), String.t() | nil) :: String.t() | nil | :no_worker_capacity + def select_worker_host_for_test(%State{} = state, preferred_worker_host) do + select_worker_host(state, preferred_worker_host) + end + defp reconcile_running_issue_states([], state, _active_states, _terminal_states), do: state defp reconcile_running_issue_states([issue | rest], state, active_states, terminal_states) do @@ -392,9 +419,10 @@ defmodule SymphonyElixir.Orchestrator do %{pid: pid, ref: ref, identifier: identifier} = running_entry -> state = record_session_completion_totals(state, running_entry) + worker_host = Map.get(running_entry, :worker_host) if cleanup_workspace do - cleanup_issue_workspace(identifier) + cleanup_issue_workspace(identifier, worker_host) end if is_pid(pid) do @@ -534,7 +562,8 @@ defmodule SymphonyElixir.Orchestrator do !MapSet.member?(claimed, issue.id) and !Map.has_key?(running, issue.id) and available_slots(state) > 0 and - state_slots_available?(issue, running) + state_slots_available?(issue, running) and + worker_slots_available?(state) end defp should_dispatch_issue?(_issue, _state, _active_states, _terminal_states), do: false @@ -628,10 +657,10 @@ defmodule SymphonyElixir.Orchestrator do |> MapSet.new() end - defp dispatch_issue(%State{} = state, issue, attempt \\ nil) do + defp dispatch_issue(%State{} = state, issue, attempt \\ nil, preferred_worker_host \\ nil) do case revalidate_issue_for_dispatch(issue, &Tracker.fetch_issue_states_by_ids/1, terminal_state_set()) do {:ok, %Issue{} = refreshed_issue} -> - do_dispatch_issue(state, refreshed_issue, attempt) + do_dispatch_issue(state, refreshed_issue, attempt, preferred_worker_host) {:skip, :missing} -> Logger.info("Skipping dispatch; issue no longer active or visible: #{issue_context(issue)}") @@ -648,16 +677,27 @@ defmodule SymphonyElixir.Orchestrator do end end - defp do_dispatch_issue(%State{} = state, issue, attempt) do + defp do_dispatch_issue(%State{} = state, issue, attempt, preferred_worker_host) do recipient = self() + case select_worker_host(state, preferred_worker_host) do + :no_worker_capacity -> + Logger.debug("No SSH worker slots available for #{issue_context(issue)} preferred_worker_host=#{inspect(preferred_worker_host)}") + state + + worker_host -> + spawn_issue_on_worker_host(state, issue, attempt, recipient, worker_host) + end + end + + defp spawn_issue_on_worker_host(%State{} = state, issue, attempt, recipient, worker_host) do case Task.Supervisor.start_child(SymphonyElixir.TaskSupervisor, fn -> - AgentRunner.run(issue, recipient, attempt: attempt) + AgentRunner.run(issue, recipient, attempt: attempt, worker_host: worker_host) end) do {:ok, pid} -> ref = Process.monitor(pid) - Logger.info("Dispatching issue to agent: #{issue_context(issue)} pid=#{inspect(pid)} attempt=#{inspect(attempt)}") + Logger.info("Dispatching issue to agent: #{issue_context(issue)} pid=#{inspect(pid)} attempt=#{inspect(attempt)} worker_host=#{worker_host || "local"}") running = Map.put(state.running, issue.id, %{ @@ -665,6 +705,8 @@ defmodule SymphonyElixir.Orchestrator do ref: ref, identifier: issue.identifier, issue: issue, + worker_host: worker_host, + workspace_path: nil, session_id: nil, last_codex_message: nil, last_codex_timestamp: nil, @@ -694,7 +736,8 @@ defmodule SymphonyElixir.Orchestrator do schedule_issue_retry(state, issue.id, next_attempt, %{ identifier: issue.identifier, - error: "failed to spawn agent: #{inspect(reason)}" + error: "failed to spawn agent: #{inspect(reason)}", + worker_host: worker_host }) end end @@ -737,6 +780,8 @@ defmodule SymphonyElixir.Orchestrator do due_at_ms = System.monotonic_time(:millisecond) + delay_ms identifier = pick_retry_identifier(issue_id, previous_retry, metadata) error = pick_retry_error(previous_retry, metadata) + worker_host = pick_retry_worker_host(previous_retry, metadata) + workspace_path = pick_retry_workspace_path(previous_retry, metadata) if is_reference(old_timer) do Process.cancel_timer(old_timer) @@ -757,7 +802,9 @@ defmodule SymphonyElixir.Orchestrator do retry_token: retry_token, due_at_ms: due_at_ms, identifier: identifier, - error: error + error: error, + worker_host: worker_host, + workspace_path: workspace_path }) } end @@ -767,7 +814,9 @@ defmodule SymphonyElixir.Orchestrator do %{attempt: attempt, retry_token: ^retry_token} = retry_entry -> metadata = %{ identifier: Map.get(retry_entry, :identifier), - error: Map.get(retry_entry, :error) + error: Map.get(retry_entry, :error), + worker_host: Map.get(retry_entry, :worker_host), + workspace_path: Map.get(retry_entry, :workspace_path) } {:ok, attempt, metadata, %{state | retry_attempts: Map.delete(state.retry_attempts, issue_id)}} @@ -804,7 +853,7 @@ defmodule SymphonyElixir.Orchestrator do terminal_issue_state?(issue.state, terminal_states) -> Logger.info("Issue state is terminal: issue_id=#{issue_id} issue_identifier=#{issue.identifier} state=#{issue.state}; removing associated workspace") - cleanup_issue_workspace(issue.identifier) + cleanup_issue_workspace(issue.identifier, metadata[:worker_host]) {:noreply, release_issue_claim(state, issue_id)} retry_candidate_issue?(issue, terminal_states) -> @@ -822,11 +871,13 @@ defmodule SymphonyElixir.Orchestrator do {:noreply, release_issue_claim(state, issue_id)} end - defp cleanup_issue_workspace(identifier) when is_binary(identifier) do - Workspace.remove_issue_workspaces(identifier) + defp cleanup_issue_workspace(identifier, worker_host \\ nil) + + defp cleanup_issue_workspace(identifier, worker_host) when is_binary(identifier) do + Workspace.remove_issue_workspaces(identifier, worker_host) end - defp cleanup_issue_workspace(_identifier), do: :ok + defp cleanup_issue_workspace(_identifier, _worker_host), do: :ok defp run_terminal_workspace_cleanup do case Tracker.fetch_issues_by_states(Config.settings!().tracker.terminal_states) do @@ -851,8 +902,9 @@ defmodule SymphonyElixir.Orchestrator do defp handle_active_retry(state, issue, attempt, metadata) do if retry_candidate_issue?(issue, terminal_state_set()) and - dispatch_slots_available?(issue, state) do - {:noreply, dispatch_issue(state, issue, attempt)} + dispatch_slots_available?(issue, state) and + worker_slots_available?(state, metadata[:worker_host]) do + {:noreply, dispatch_issue(state, issue, attempt, metadata[:worker_host])} else Logger.debug("No available slots for retrying #{issue_context(issue)}; retrying again") @@ -904,6 +956,82 @@ defmodule SymphonyElixir.Orchestrator do metadata[:error] || Map.get(previous_retry, :error) end + defp pick_retry_worker_host(previous_retry, metadata) do + metadata[:worker_host] || Map.get(previous_retry, :worker_host) + end + + defp pick_retry_workspace_path(previous_retry, metadata) do + metadata[:workspace_path] || Map.get(previous_retry, :workspace_path) + end + + defp maybe_put_runtime_value(running_entry, _key, nil), do: running_entry + + defp maybe_put_runtime_value(running_entry, key, value) when is_map(running_entry) do + Map.put(running_entry, key, value) + end + + defp select_worker_host(%State{} = state, preferred_worker_host) do + case Config.settings!().worker.ssh_hosts do + [] -> + nil + + hosts -> + available_hosts = Enum.filter(hosts, &worker_host_slots_available?(state, &1)) + + cond do + available_hosts == [] -> + :no_worker_capacity + + preferred_worker_host_available?(preferred_worker_host, available_hosts) -> + preferred_worker_host + + true -> + least_loaded_worker_host(state, available_hosts) + end + end + end + + defp preferred_worker_host_available?(preferred_worker_host, hosts) + when is_binary(preferred_worker_host) and is_list(hosts) do + preferred_worker_host != "" and preferred_worker_host in hosts + end + + defp preferred_worker_host_available?(_preferred_worker_host, _hosts), do: false + + defp least_loaded_worker_host(%State{} = state, hosts) when is_list(hosts) do + hosts + |> Enum.with_index() + |> Enum.min_by(fn {host, index} -> + {running_worker_host_count(state.running, host), index} + end) + |> elem(0) + end + + defp running_worker_host_count(running, worker_host) when is_map(running) and is_binary(worker_host) do + Enum.count(running, fn + {_issue_id, %{worker_host: ^worker_host}} -> true + _ -> false + end) + end + + defp worker_slots_available?(%State{} = state) do + select_worker_host(state, nil) != :no_worker_capacity + end + + defp worker_slots_available?(%State{} = state, preferred_worker_host) do + select_worker_host(state, preferred_worker_host) != :no_worker_capacity + end + + defp worker_host_slots_available?(%State{} = state, worker_host) when is_binary(worker_host) do + case Config.settings!().worker.max_concurrent_agents_per_host do + limit when is_integer(limit) and limit > 0 -> + running_worker_host_count(state.running, worker_host) < limit + + _ -> + true + end + end + defp find_issue_by_id(issues, issue_id) when is_binary(issue_id) do Enum.find(issues, fn %Issue{id: ^issue_id} -> @@ -982,6 +1110,8 @@ defmodule SymphonyElixir.Orchestrator do issue_id: issue_id, identifier: metadata.identifier, state: metadata.issue.state, + worker_host: Map.get(metadata, :worker_host), + workspace_path: Map.get(metadata, :workspace_path), session_id: metadata.session_id, codex_app_server_pid: metadata.codex_app_server_pid, codex_input_tokens: metadata.codex_input_tokens, @@ -1004,7 +1134,9 @@ defmodule SymphonyElixir.Orchestrator do attempt: attempt, due_in_ms: max(0, due_at_ms - now_ms), identifier: Map.get(retry, :identifier), - error: Map.get(retry, :error) + error: Map.get(retry, :error), + worker_host: Map.get(retry, :worker_host), + workspace_path: Map.get(retry, :workspace_path) } end) diff --git a/elixir/lib/symphony_elixir/ssh.ex b/elixir/lib/symphony_elixir/ssh.ex new file mode 100644 index 0000000000..0493adb083 --- /dev/null +++ b/elixir/lib/symphony_elixir/ssh.ex @@ -0,0 +1,100 @@ +defmodule SymphonyElixir.SSH do + @moduledoc false + + @spec run(String.t(), String.t(), keyword()) :: {:ok, {String.t(), non_neg_integer()}} | {:error, term()} + def run(host, command, opts \\ []) when is_binary(host) and is_binary(command) do + with {:ok, executable} <- ssh_executable() do + {:ok, System.cmd(executable, ssh_args(host, command), opts)} + end + end + + @spec start_port(String.t(), String.t(), keyword()) :: {:ok, port()} | {:error, term()} + def start_port(host, command, opts \\ []) when is_binary(host) and is_binary(command) do + with {:ok, executable} <- ssh_executable() do + line_bytes = Keyword.get(opts, :line) + + port_opts = + [ + :binary, + :exit_status, + :stderr_to_stdout, + args: Enum.map(ssh_args(host, command), &String.to_charlist/1) + ] + |> maybe_put_line_option(line_bytes) + + {:ok, Port.open({:spawn_executable, String.to_charlist(executable)}, port_opts)} + end + end + + @spec remote_shell_command(String.t()) :: String.t() + def remote_shell_command(command) when is_binary(command) do + "bash -lc " <> shell_escape(command) + end + + defp ssh_executable do + case System.find_executable("ssh") do + nil -> {:error, :ssh_not_found} + executable -> {:ok, executable} + end + end + + defp ssh_args(host, command) do + %{destination: destination, port: port} = parse_target(host) + + [] + |> maybe_put_config() + |> Kernel.++(["-T"]) + |> maybe_put_port(port) + |> Kernel.++([destination, remote_shell_command(command)]) + end + + defp maybe_put_line_option(port_opts, nil), do: port_opts + defp maybe_put_line_option(port_opts, line_bytes), do: Keyword.put(port_opts, :line, line_bytes) + + defp maybe_put_config(args) do + case System.get_env("SYMPHONY_SSH_CONFIG") do + config_path when is_binary(config_path) and config_path != "" -> + args ++ ["-F", config_path] + + _ -> + args + end + end + + defp maybe_put_port(args, nil), do: args + defp maybe_put_port(args, port), do: args ++ ["-p", port] + + defp parse_target(target) when is_binary(target) do + trimmed_target = String.trim(target) + + # OpenSSH does not interpret bare "host:port" as "host + port"; it treats the + # whole value as a hostname and leaves the port at 22. We split that shorthand + # here so worker config can use "localhost:2222" without requiring ssh:// URIs. + case Regex.run(~r/^(.*):(\d+)$/, trimmed_target, capture: :all_but_first) do + [destination, port] -> + if valid_port_destination?(destination) do + %{destination: destination, port: port} + else + %{destination: trimmed_target, port: nil} + end + + _ -> + %{destination: trimmed_target, port: nil} + end + end + + defp valid_port_destination?(destination) when is_binary(destination) do + destination != "" and + (not String.contains?(destination, ":") or bracketed_host?(destination)) + end + + defp bracketed_host?(destination) when is_binary(destination) do + # IPv6 literals contain ":" already, so we only accept additional ":port" + # parsing when the host is explicitly bracketed, e.g. "[::1]:2222". + String.contains?(destination, "[") and String.contains?(destination, "]") + end + + defp shell_escape(value) when is_binary(value) do + "'" <> String.replace(value, "'", "'\"'\"'") <> "'" + end +end diff --git a/elixir/lib/symphony_elixir/workspace.ex b/elixir/lib/symphony_elixir/workspace.ex index 539976e93d..14e29da402 100644 --- a/elixir/lib/symphony_elixir/workspace.ex +++ b/elixir/lib/symphony_elixir/workspace.ex @@ -4,35 +4,37 @@ defmodule SymphonyElixir.Workspace do """ require Logger - alias SymphonyElixir.{Config, PathSafety} + alias SymphonyElixir.{Config, PathSafety, SSH} - @excluded_entries MapSet.new([".elixir_ls", "tmp"]) + @remote_workspace_marker "__SYMPHONY_WORKSPACE__" - @spec create_for_issue(map() | String.t() | nil) :: {:ok, Path.t()} | {:error, term()} - def create_for_issue(issue_or_identifier) do + @type worker_host :: String.t() | nil + + @spec create_for_issue(map() | String.t() | nil, worker_host()) :: + {:ok, Path.t()} | {:error, term()} + def create_for_issue(issue_or_identifier, worker_host \\ nil) do issue_context = issue_context(issue_or_identifier) try do safe_id = safe_identifier(issue_context.issue_identifier) - with {:ok, workspace} <- workspace_path_for_issue(safe_id), - :ok <- validate_workspace_path(workspace), - {:ok, created?} <- ensure_workspace(workspace), - :ok <- maybe_run_after_create_hook(workspace, issue_context, created?) do + with {:ok, workspace} <- workspace_path_for_issue(safe_id, worker_host), + :ok <- validate_workspace_path(workspace, worker_host), + {:ok, workspace, created?} <- ensure_workspace(workspace, worker_host), + :ok <- maybe_run_after_create_hook(workspace, issue_context, created?, worker_host) do {:ok, workspace} end rescue error in [ArgumentError, ErlangError, File.Error] -> - Logger.error("Workspace creation failed #{issue_log_context(issue_context)} error=#{Exception.message(error)}") + Logger.error("Workspace creation failed #{issue_log_context(issue_context)} worker_host=#{worker_host_for_log(worker_host)} error=#{Exception.message(error)}") {:error, error} end end - defp ensure_workspace(workspace) do + defp ensure_workspace(workspace, nil) do cond do File.dir?(workspace) -> - clean_tmp_artifacts(workspace) - {:ok, false} + {:ok, workspace, false} File.exists?(workspace) -> File.rm_rf!(workspace) @@ -43,19 +45,55 @@ defmodule SymphonyElixir.Workspace do end end + defp ensure_workspace(workspace, worker_host) when is_binary(worker_host) do + script = + [ + "set -eu", + remote_shell_assign("workspace", workspace), + "if [ -d \"$workspace\" ]; then", + " created=0", + "elif [ -e \"$workspace\" ]; then", + " rm -rf \"$workspace\"", + " mkdir -p \"$workspace\"", + " created=1", + "else", + " mkdir -p \"$workspace\"", + " created=1", + "fi", + "cd \"$workspace\"", + "printf '%s\\t%s\\t%s\\n' '#{@remote_workspace_marker}' \"$created\" \"$(pwd -P)\"" + ] + |> Enum.reject(&(&1 == "")) + |> Enum.join("\n") + + case run_remote_command(worker_host, script, Config.settings!().hooks.timeout_ms) do + {:ok, {output, 0}} -> + parse_remote_workspace_output(output) + + {:ok, {output, status}} -> + {:error, {:workspace_prepare_failed, worker_host, status, output}} + + {:error, reason} -> + {:error, reason} + end + end + defp create_workspace(workspace) do File.rm_rf!(workspace) File.mkdir_p!(workspace) - {:ok, true} + {:ok, workspace, true} end @spec remove(Path.t()) :: {:ok, [String.t()]} | {:error, term(), String.t()} - def remove(workspace) do + def remove(workspace), do: remove(workspace, nil) + + @spec remove(Path.t(), worker_host()) :: {:ok, [String.t()]} | {:error, term(), String.t()} + def remove(workspace, nil) do case File.exists?(workspace) do true -> - case validate_workspace_path(workspace) do + case validate_workspace_path(workspace, nil) do :ok -> - maybe_run_before_remove_hook(workspace) + maybe_run_before_remove_hook(workspace, nil) File.rm_rf(workspace) {:error, reason} -> @@ -67,24 +105,67 @@ defmodule SymphonyElixir.Workspace do end end + def remove(workspace, worker_host) when is_binary(worker_host) do + maybe_run_before_remove_hook(workspace, worker_host) + + script = + [ + remote_shell_assign("workspace", workspace), + "rm -rf \"$workspace\"" + ] + |> Enum.join("\n") + + case run_remote_command(worker_host, script, Config.settings!().hooks.timeout_ms) do + {:ok, {_output, 0}} -> + {:ok, []} + + {:ok, {output, status}} -> + {:error, {:workspace_remove_failed, worker_host, status, output}, ""} + + {:error, reason} -> + {:error, reason, ""} + end + end + @spec remove_issue_workspaces(term()) :: :ok - def remove_issue_workspaces(identifier) when is_binary(identifier) do + def remove_issue_workspaces(identifier), do: remove_issue_workspaces(identifier, nil) + + @spec remove_issue_workspaces(term(), worker_host()) :: :ok + def remove_issue_workspaces(identifier, worker_host) when is_binary(identifier) and is_binary(worker_host) do safe_id = safe_identifier(identifier) - case workspace_path_for_issue(safe_id) do - {:ok, workspace} -> remove(workspace) + case workspace_path_for_issue(safe_id, worker_host) do + {:ok, workspace} -> remove(workspace, worker_host) {:error, _reason} -> :ok end :ok end - def remove_issue_workspaces(_identifier) do + def remove_issue_workspaces(identifier, nil) when is_binary(identifier) do + safe_id = safe_identifier(identifier) + + case Config.settings!().worker.ssh_hosts do + [] -> + case workspace_path_for_issue(safe_id, nil) do + {:ok, workspace} -> remove(workspace, nil) + {:error, _reason} -> :ok + end + + worker_hosts -> + Enum.each(worker_hosts, &remove_issue_workspaces(identifier, &1)) + end + + :ok + end + + def remove_issue_workspaces(_identifier, _worker_host) do :ok end - @spec run_before_run_hook(Path.t(), map() | String.t() | nil) :: :ok | {:error, term()} - def run_before_run_hook(workspace, issue_or_identifier) when is_binary(workspace) do + @spec run_before_run_hook(Path.t(), map() | String.t() | nil, worker_host()) :: + :ok | {:error, term()} + def run_before_run_hook(workspace, issue_or_identifier, worker_host \\ nil) when is_binary(workspace) do issue_context = issue_context(issue_or_identifier) hooks = Config.settings!().hooks @@ -93,12 +174,12 @@ defmodule SymphonyElixir.Workspace do :ok command -> - run_hook(command, workspace, issue_context, "before_run") + run_hook(command, workspace, issue_context, "before_run", worker_host) end end - @spec run_after_run_hook(Path.t(), map() | String.t() | nil) :: :ok - def run_after_run_hook(workspace, issue_or_identifier) when is_binary(workspace) do + @spec run_after_run_hook(Path.t(), map() | String.t() | nil, worker_host()) :: :ok + def run_after_run_hook(workspace, issue_or_identifier, worker_host \\ nil) when is_binary(workspace) do issue_context = issue_context(issue_or_identifier) hooks = Config.settings!().hooks @@ -107,28 +188,26 @@ defmodule SymphonyElixir.Workspace do :ok command -> - run_hook(command, workspace, issue_context, "after_run") + run_hook(command, workspace, issue_context, "after_run", worker_host) |> ignore_hook_failure() end end - defp workspace_path_for_issue(safe_id) when is_binary(safe_id) do + defp workspace_path_for_issue(safe_id, nil) when is_binary(safe_id) do Config.settings!().workspace.root |> Path.join(safe_id) |> PathSafety.canonicalize() end - defp safe_identifier(identifier) do - String.replace(identifier || "issue", ~r/[^a-zA-Z0-9._-]/, "_") + defp workspace_path_for_issue(safe_id, worker_host) when is_binary(safe_id) and is_binary(worker_host) do + {:ok, Path.join(Config.settings!().workspace.root, safe_id)} end - defp clean_tmp_artifacts(workspace) do - Enum.each(MapSet.to_list(@excluded_entries), fn entry -> - File.rm_rf(Path.join(workspace, entry)) - end) + defp safe_identifier(identifier) do + String.replace(identifier || "issue", ~r/[^a-zA-Z0-9._-]/, "_") end - defp maybe_run_after_create_hook(workspace, issue_context, created?) do + defp maybe_run_after_create_hook(workspace, issue_context, created?, worker_host) do hooks = Config.settings!().hooks case created? do @@ -138,7 +217,7 @@ defmodule SymphonyElixir.Workspace do :ok command -> - run_hook(command, workspace, issue_context, "after_create") + run_hook(command, workspace, issue_context, "after_create", worker_host) end false -> @@ -146,7 +225,7 @@ defmodule SymphonyElixir.Workspace do end end - defp maybe_run_before_remove_hook(workspace) do + defp maybe_run_before_remove_hook(workspace, nil) do hooks = Config.settings!().hooks case File.dir?(workspace) do @@ -160,7 +239,8 @@ defmodule SymphonyElixir.Workspace do command, workspace, %{issue_id: nil, issue_identifier: Path.basename(workspace)}, - "before_remove" + "before_remove", + nil ) |> ignore_hook_failure() end @@ -170,13 +250,51 @@ defmodule SymphonyElixir.Workspace do end end + defp maybe_run_before_remove_hook(workspace, worker_host) when is_binary(worker_host) do + hooks = Config.settings!().hooks + + case hooks.before_remove do + nil -> + :ok + + command -> + script = + [ + remote_shell_assign("workspace", workspace), + "if [ -d \"$workspace\" ]; then", + " cd \"$workspace\"", + " #{command}", + "fi" + ] + |> Enum.join("\n") + + run_remote_command(worker_host, script, Config.settings!().hooks.timeout_ms) + |> case do + {:ok, {output, status}} -> + handle_hook_command_result( + {output, status}, + workspace, + %{issue_id: nil, issue_identifier: Path.basename(workspace)}, + "before_remove" + ) + + {:error, {:workspace_hook_timeout, "before_remove", _timeout_ms} = reason} -> + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + |> ignore_hook_failure() + end + end + defp ignore_hook_failure(:ok), do: :ok defp ignore_hook_failure({:error, _reason}), do: :ok - defp run_hook(command, workspace, issue_context, hook_name) do + defp run_hook(command, workspace, issue_context, hook_name, nil) do timeout_ms = Config.settings!().hooks.timeout_ms - Logger.info("Running workspace hook hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace}") + Logger.info("Running workspace hook hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace} worker_host=local") task = Task.async(fn -> @@ -190,12 +308,29 @@ defmodule SymphonyElixir.Workspace do nil -> Task.shutdown(task, :brutal_kill) - Logger.warning("Workspace hook timed out hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace} timeout_ms=#{timeout_ms}") + Logger.warning("Workspace hook timed out hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace} worker_host=local timeout_ms=#{timeout_ms}") {:error, {:workspace_hook_timeout, hook_name, timeout_ms}} end end + defp run_hook(command, workspace, issue_context, hook_name, worker_host) when is_binary(worker_host) do + timeout_ms = Config.settings!().hooks.timeout_ms + + Logger.info("Running workspace hook hook=#{hook_name} #{issue_log_context(issue_context)} workspace=#{workspace} worker_host=#{worker_host}") + + case run_remote_command(worker_host, "cd #{shell_escape(workspace)} && #{command}", timeout_ms) do + {:ok, cmd_result} -> + handle_hook_command_result(cmd_result, workspace, issue_context, hook_name) + + {:error, {:workspace_hook_timeout, ^hook_name, _timeout_ms} = reason} -> + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + end + defp handle_hook_command_result({_output, 0}, _workspace, _issue_id, _hook_name) do :ok end @@ -220,7 +355,7 @@ defmodule SymphonyElixir.Workspace do end end - defp validate_workspace_path(workspace) when is_binary(workspace) do + defp validate_workspace_path(workspace, nil) when is_binary(workspace) do expanded_workspace = Path.expand(workspace) expanded_root = Path.expand(Config.settings!().workspace.root) expanded_root_prefix = expanded_root <> "/" @@ -248,6 +383,79 @@ defmodule SymphonyElixir.Workspace do end end + defp validate_workspace_path(workspace, worker_host) + when is_binary(workspace) and is_binary(worker_host) do + cond do + String.trim(workspace) == "" -> + {:error, {:workspace_path_unreadable, workspace, :empty}} + + String.contains?(workspace, ["\n", "\r", <<0>>]) -> + {:error, {:workspace_path_unreadable, workspace, :invalid_characters}} + + true -> + :ok + end + end + + defp remote_shell_assign(variable_name, raw_path) + when is_binary(variable_name) and is_binary(raw_path) do + [ + "#{variable_name}=#{shell_escape(raw_path)}", + "case \"$#{variable_name}\" in", + " '~') #{variable_name}=\"$HOME\" ;;", + " '~/'*) " <> variable_name <> "=\"$HOME/${" <> variable_name <> "#~/}\" ;;", + "esac" + ] + |> Enum.join("\n") + end + + defp parse_remote_workspace_output(output) do + lines = String.split(IO.iodata_to_binary(output), "\n", trim: true) + + payload = + Enum.find_value(lines, fn line -> + case String.split(line, "\t", parts: 3) do + [@remote_workspace_marker, created, path] when created in ["0", "1"] and path != "" -> + {created == "1", path} + + _ -> + nil + end + end) + + case payload do + {created?, workspace} when is_boolean(created?) and is_binary(workspace) -> + {:ok, workspace, created?} + + _ -> + {:error, {:workspace_prepare_failed, :invalid_output, output}} + end + end + + defp run_remote_command(worker_host, script, timeout_ms) + when is_binary(worker_host) and is_binary(script) and is_integer(timeout_ms) and timeout_ms > 0 do + task = + Task.async(fn -> + SSH.run(worker_host, script, stderr_to_stdout: true) + end) + + case Task.yield(task, timeout_ms) do + {:ok, result} -> + result + + nil -> + Task.shutdown(task, :brutal_kill) + {:error, {:workspace_hook_timeout, "remote_command", timeout_ms}} + end + end + + defp shell_escape(value) when is_binary(value) do + "'" <> String.replace(value, "'", "'\"'\"'") <> "'" + end + + defp worker_host_for_log(nil), do: "local" + defp worker_host_for_log(worker_host), do: worker_host + defp issue_context(%{id: issue_id, identifier: identifier}) do %{ issue_id: issue_id, diff --git a/elixir/lib/symphony_elixir_web/presenter.ex b/elixir/lib/symphony_elixir_web/presenter.ex index dc78ab3234..1063cf7a64 100644 --- a/elixir/lib/symphony_elixir_web/presenter.ex +++ b/elixir/lib/symphony_elixir_web/presenter.ex @@ -66,7 +66,8 @@ defmodule SymphonyElixirWeb.Presenter do issue_id: issue_id_from_entries(running, retry), status: issue_status(running, retry), workspace: %{ - path: Path.join(Config.settings!().workspace.root, issue_identifier) + path: workspace_path(issue_identifier, running, retry), + host: workspace_host(running, retry) }, attempts: %{ restart_count: restart_count(retry), @@ -99,6 +100,8 @@ defmodule SymphonyElixirWeb.Presenter do issue_id: entry.issue_id, issue_identifier: entry.identifier, state: entry.state, + worker_host: Map.get(entry, :worker_host), + workspace_path: Map.get(entry, :workspace_path), session_id: entry.session_id, turn_count: Map.get(entry, :turn_count, 0), last_event: entry.last_codex_event, @@ -119,12 +122,16 @@ defmodule SymphonyElixirWeb.Presenter do issue_identifier: entry.identifier, attempt: entry.attempt, due_at: due_at_iso8601(entry.due_in_ms), - error: entry.error + error: entry.error, + worker_host: Map.get(entry, :worker_host), + workspace_path: Map.get(entry, :workspace_path) } end defp running_issue_payload(running) do %{ + worker_host: Map.get(running, :worker_host), + workspace_path: Map.get(running, :workspace_path), session_id: running.session_id, turn_count: Map.get(running, :turn_count, 0), state: running.state, @@ -144,10 +151,22 @@ defmodule SymphonyElixirWeb.Presenter do %{ attempt: retry.attempt, due_at: due_at_iso8601(retry.due_in_ms), - error: retry.error + error: retry.error, + worker_host: Map.get(retry, :worker_host), + workspace_path: Map.get(retry, :workspace_path) } end + defp workspace_path(issue_identifier, running, retry) do + (running && Map.get(running, :workspace_path)) || + (retry && Map.get(retry, :workspace_path)) || + Path.join(Config.settings!().workspace.root, issue_identifier) + end + + defp workspace_host(running, retry) do + (running && Map.get(running, :worker_host)) || (retry && Map.get(retry, :worker_host)) + end + defp recent_events_payload(running) do [ %{ diff --git a/elixir/test/support/live_e2e_docker/Dockerfile b/elixir/test/support/live_e2e_docker/Dockerfile new file mode 100644 index 0000000000..974625c1cd --- /dev/null +++ b/elixir/test/support/live_e2e_docker/Dockerfile @@ -0,0 +1,22 @@ +FROM node:20-bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + git \ + openssh-server \ + python3 \ + ripgrep \ + && rm -rf /var/lib/apt/lists/* + +RUN install -d -m 700 /root/.ssh /root/.codex /run/symphony/ssh /var/run/sshd + +RUN npm install --global @openai/codex + +COPY symphony-live-worker.conf /etc/ssh/sshd_config.d/symphony-live-worker.conf +COPY live_worker_entrypoint.sh /usr/local/bin/symphony-live-worker +RUN chmod 755 /usr/local/bin/symphony-live-worker + +EXPOSE 22 + +ENTRYPOINT ["/usr/local/bin/symphony-live-worker"] diff --git a/elixir/test/support/live_e2e_docker/docker-compose.yml b/elixir/test/support/live_e2e_docker/docker-compose.yml new file mode 100644 index 0000000000..31584538d9 --- /dev/null +++ b/elixir/test/support/live_e2e_docker/docker-compose.yml @@ -0,0 +1,20 @@ +services: + worker1: + build: + context: . + dockerfile: Dockerfile + ports: + - "${SYMPHONY_LIVE_DOCKER_WORKER_1_PORT}:22" + volumes: + - ${SYMPHONY_LIVE_DOCKER_AUTHORIZED_KEY}:/run/symphony/ssh/authorized_key.pub:ro + - ${SYMPHONY_LIVE_DOCKER_AUTH_JSON}:/root/.codex/auth.json:ro + + worker2: + build: + context: . + dockerfile: Dockerfile + ports: + - "${SYMPHONY_LIVE_DOCKER_WORKER_2_PORT}:22" + volumes: + - ${SYMPHONY_LIVE_DOCKER_AUTHORIZED_KEY}:/run/symphony/ssh/authorized_key.pub:ro + - ${SYMPHONY_LIVE_DOCKER_AUTH_JSON}:/root/.codex/auth.json:ro diff --git a/elixir/test/support/live_e2e_docker/live_worker_entrypoint.sh b/elixir/test/support/live_e2e_docker/live_worker_entrypoint.sh new file mode 100644 index 0000000000..3b70e6f4e4 --- /dev/null +++ b/elixir/test/support/live_e2e_docker/live_worker_entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +install -d -m 700 /root/.ssh /root/.codex + +if [ ! -s /run/symphony/ssh/authorized_key.pub ]; then + echo "missing authorized key at /run/symphony/ssh/authorized_key.pub" >&2 + exit 1 +fi + +install -m 600 /run/symphony/ssh/authorized_key.pub /root/.ssh/authorized_keys + +exec /usr/sbin/sshd -D -e diff --git a/elixir/test/support/live_e2e_docker/symphony-live-worker.conf b/elixir/test/support/live_e2e_docker/symphony-live-worker.conf new file mode 100644 index 0000000000..45cc12dc6d --- /dev/null +++ b/elixir/test/support/live_e2e_docker/symphony-live-worker.conf @@ -0,0 +1,7 @@ +PubkeyAuthentication yes +PasswordAuthentication no +KbdInteractiveAuthentication no +ChallengeResponseAuthentication no +UsePAM no +PermitRootLogin yes +AuthorizedKeysFile .ssh/authorized_keys diff --git a/elixir/test/support/test_support.exs b/elixir/test/support/test_support.exs index bea30f2cf1..484c1cae72 100644 --- a/elixir/test/support/test_support.exs +++ b/elixir/test/support/test_support.exs @@ -101,6 +101,8 @@ defmodule SymphonyElixir.TestSupport do tracker_terminal_states: ["Closed", "Cancelled", "Canceled", "Duplicate", "Done"], poll_interval_ms: 30_000, workspace_root: Path.join(System.tmp_dir!(), "symphony_workspaces"), + worker_ssh_hosts: [], + worker_max_concurrent_agents_per_host: nil, max_concurrent_agents: 10, max_turns: 20, max_retry_backoff_ms: 300_000, @@ -136,6 +138,8 @@ defmodule SymphonyElixir.TestSupport do tracker_terminal_states = Keyword.get(config, :tracker_terminal_states) poll_interval_ms = Keyword.get(config, :poll_interval_ms) workspace_root = Keyword.get(config, :workspace_root) + worker_ssh_hosts = Keyword.get(config, :worker_ssh_hosts) + worker_max_concurrent_agents_per_host = Keyword.get(config, :worker_max_concurrent_agents_per_host) max_concurrent_agents = Keyword.get(config, :max_concurrent_agents) max_turns = Keyword.get(config, :max_turns) max_retry_backoff_ms = Keyword.get(config, :max_retry_backoff_ms) @@ -174,6 +178,7 @@ defmodule SymphonyElixir.TestSupport do " interval_ms: #{yaml_value(poll_interval_ms)}", "workspace:", " root: #{yaml_value(workspace_root)}", + worker_yaml(worker_ssh_hosts, worker_max_concurrent_agents_per_host), "agent:", " max_concurrent_agents: #{yaml_value(max_concurrent_agents)}", " max_turns: #{yaml_value(max_turns)}", @@ -235,6 +240,21 @@ defmodule SymphonyElixir.TestSupport do |> Enum.join("\n") end + defp worker_yaml(ssh_hosts, max_concurrent_agents_per_host) + when ssh_hosts in [nil, []] and is_nil(max_concurrent_agents_per_host), + do: nil + + defp worker_yaml(ssh_hosts, max_concurrent_agents_per_host) do + [ + "worker:", + ssh_hosts not in [nil, []] && " ssh_hosts: #{yaml_value(ssh_hosts)}", + !is_nil(max_concurrent_agents_per_host) && + " max_concurrent_agents_per_host: #{yaml_value(max_concurrent_agents_per_host)}" + ] + |> Enum.reject(&(&1 in [nil, false])) + |> Enum.join("\n") + end + defp observability_yaml(enabled, refresh_ms, render_interval_ms) do [ "observability:", diff --git a/elixir/test/symphony_elixir/app_server_test.exs b/elixir/test/symphony_elixir/app_server_test.exs index 3b98c44343..b7fab1521c 100644 --- a/elixir/test/symphony_elixir/app_server_test.exs +++ b/elixir/test/symphony_elixir/app_server_test.exs @@ -825,9 +825,8 @@ defmodule SymphonyElixir.AppServerTest do payload["id"] == 101 and get_in(payload, ["result", "success"]) == false and - get_in(payload, ["result", "contentItems", Access.at(0), "type"]) == "inputText" and String.contains?( - get_in(payload, ["result", "contentItems", Access.at(0), "text"]), + get_in(payload, ["result", "output"]), "Unsupported dynamic tool" ) else @@ -950,7 +949,7 @@ defmodule SymphonyElixir.AppServerTest do payload["id"] == 102 and get_in(payload, ["result", "success"]) == true and - get_in(payload, ["result", "contentItems", Access.at(0), "text"]) == + get_in(payload, ["result", "output"]) == ~s({"data":{"viewer":{"id":"usr_123"}}}) else false @@ -1199,4 +1198,136 @@ defmodule SymphonyElixir.AppServerTest do File.rm_rf(test_root) end end + + test "app server launches over ssh for remote workers" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-app-server-remote-ssh-#{System.unique_integer([:positive])}" + ) + + previous_path = System.get_env("PATH") + previous_trace = System.get_env("SYMP_TEST_SSH_TRACE") + + on_exit(fn -> + restore_env("PATH", previous_path) + restore_env("SYMP_TEST_SSH_TRACE", previous_trace) + end) + + try do + trace_file = Path.join(test_root, "ssh.trace") + fake_ssh = Path.join(test_root, "ssh") + remote_workspace = "/remote/workspaces/MT-REMOTE" + + File.mkdir_p!(test_root) + System.put_env("SYMP_TEST_SSH_TRACE", trace_file) + System.put_env("PATH", test_root <> ":" <> (previous_path || "")) + + File.write!(fake_ssh, """ + #!/bin/sh + trace_file="${SYMP_TEST_SSH_TRACE:-/tmp/symphony-fake-ssh.trace}" + count=0 + printf 'ARGV:%s\\n' "$*" >> "$trace_file" + + while IFS= read -r line; do + count=$((count + 1)) + printf 'JSON:%s\\n' "$line" >> "$trace_file" + + case "$count" in + 1) + printf '%s\\n' '{"id":1,"result":{}}' + ;; + 2) + printf '%s\\n' '{"id":2,"result":{"thread":{"id":"thread-remote"}}}' + ;; + 3) + printf '%s\\n' '{"id":3,"result":{"turn":{"id":"turn-remote"}}}' + ;; + 4) + printf '%s\\n' '{"method":"turn/completed"}' + exit 0 + ;; + *) + exit 0 + ;; + esac + done + """) + + File.chmod!(fake_ssh, 0o755) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: "/remote/workspaces", + codex_command: "fake-remote-codex app-server" + ) + + issue = %Issue{ + id: "issue-remote", + identifier: "MT-REMOTE", + title: "Run remote app server", + description: "Validate ssh-backed codex startup", + state: "In Progress", + url: "https://example.org/issues/MT-REMOTE", + labels: ["backend"] + } + + assert {:ok, _result} = + AppServer.run( + remote_workspace, + "Run remote worker", + issue, + worker_host: "worker-01:2200" + ) + + trace = File.read!(trace_file) + lines = String.split(trace, "\n", trim: true) + + assert argv_line = Enum.find(lines, &String.starts_with?(&1, "ARGV:")) + assert argv_line =~ "-T -p 2200 worker-01 bash -lc" + assert argv_line =~ "cd " + assert argv_line =~ remote_workspace + assert argv_line =~ "exec " + assert argv_line =~ "fake-remote-codex app-server" + + expected_turn_policy = %{ + "type" => "workspaceWrite", + "writableRoots" => [remote_workspace], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + + assert Enum.any?(lines, fn line -> + if String.starts_with?(line, "JSON:") do + line + |> String.trim_leading("JSON:") + |> Jason.decode!() + |> then(fn payload -> + payload["method"] == "thread/start" && + get_in(payload, ["params", "cwd"]) == remote_workspace + end) + else + false + end + end) + + assert Enum.any?(lines, fn line -> + if String.starts_with?(line, "JSON:") do + line + |> String.trim_leading("JSON:") + |> Jason.decode!() + |> then(fn payload -> + payload["method"] == "turn/start" && + get_in(payload, ["params", "cwd"]) == remote_workspace && + get_in(payload, ["params", "sandboxPolicy"]) == expected_turn_policy + end) + else + false + end + end) + after + File.rm_rf(test_root) + end + end end diff --git a/elixir/test/symphony_elixir/core_test.exs b/elixir/test/symphony_elixir/core_test.exs index fa96b7b091..2e33239382 100644 --- a/elixir/test/symphony_elixir/core_test.exs +++ b/elixir/test/symphony_elixir/core_test.exs @@ -703,6 +703,53 @@ defmodule SymphonyElixir.CoreTest do assert {:noreply, ^coalesced_state} = Orchestrator.handle_info({:tick, stale_tick_token}, coalesced_state) end + test "select_worker_host_for_test skips full ssh hosts under the shared per-host cap" do + write_workflow_file!(Workflow.workflow_file_path(), + worker_ssh_hosts: ["worker-a", "worker-b"], + worker_max_concurrent_agents_per_host: 1 + ) + + state = %Orchestrator.State{ + running: %{ + "issue-1" => %{worker_host: "worker-a"} + } + } + + assert Orchestrator.select_worker_host_for_test(state, nil) == "worker-b" + end + + test "select_worker_host_for_test returns no_worker_capacity when every ssh host is full" do + write_workflow_file!(Workflow.workflow_file_path(), + worker_ssh_hosts: ["worker-a", "worker-b"], + worker_max_concurrent_agents_per_host: 1 + ) + + state = %Orchestrator.State{ + running: %{ + "issue-1" => %{worker_host: "worker-a"}, + "issue-2" => %{worker_host: "worker-b"} + } + } + + assert Orchestrator.select_worker_host_for_test(state, nil) == :no_worker_capacity + end + + test "select_worker_host_for_test keeps the preferred ssh host when it still has capacity" do + write_workflow_file!(Workflow.workflow_file_path(), + worker_ssh_hosts: ["worker-a", "worker-b"], + worker_max_concurrent_agents_per_host: 2 + ) + + state = %Orchestrator.State{ + running: %{ + "issue-1" => %{worker_host: "worker-a"}, + "issue-2" => %{worker_host: "worker-b"} + } + } + + assert Orchestrator.select_worker_host_for_test(state, "worker-a") == "worker-a" + end + defp assert_due_in_range(due_at_ms, min_remaining_ms, max_remaining_ms) do remaining_ms = due_at_ms - System.monotonic_time(:millisecond) diff --git a/elixir/test/symphony_elixir/dynamic_tool_test.exs b/elixir/test/symphony_elixir/dynamic_tool_test.exs index a5536e0338..294471ed9d 100644 --- a/elixir/test/symphony_elixir/dynamic_tool_test.exs +++ b/elixir/test/symphony_elixir/dynamic_tool_test.exs @@ -27,19 +27,19 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "type" => "inputText", - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => ~s(Unsupported dynamic tool: "not_a_real_tool".), "supportedTools" => ["linear_graphql"] } } + + assert response["contentItems"] == [ + %{ + "type" => "inputText", + "text" => response["output"] + } + ] end test "linear_graphql returns successful GraphQL responses as tool text" do @@ -61,15 +61,8 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert_received {:linear_client_called, "query Viewer { viewer { id } }", %{"includeTeams" => false}, []} assert response["success"] == true - - assert [ - %{ - "type" => "inputText", - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{"data" => %{"viewer" => %{"id" => "usr_123"}}} + assert Jason.decode!(response["output"]) == %{"data" => %{"viewer" => %{"id" => "usr_123"}}} + assert response["contentItems"] == [%{"type" => "inputText", "text" => response["output"]}] end test "linear_graphql accepts a raw GraphQL query string" do @@ -134,13 +127,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => "`linear_graphql` requires a non-empty `query` string." } @@ -159,14 +146,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "type" => "inputText", - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "data" => nil, "errors" => [%{"message" => "Unknown field `nope`"}] } @@ -197,14 +177,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "type" => "inputText", - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => "`linear_graphql` requires a non-empty `query` string." } @@ -234,13 +207,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => "`linear_graphql` expects either a GraphQL query string or an object with `query` and optional `variables`." } @@ -259,13 +226,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => "`linear_graphql.variables` must be a JSON object when provided." } @@ -282,13 +243,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert missing_token["success"] == false - assert [ - %{ - "text" => missing_token_text - } - ] = missing_token["contentItems"] - - assert Jason.decode!(missing_token_text) == %{ + assert Jason.decode!(missing_token["output"]) == %{ "error" => %{ "message" => "Symphony is missing Linear auth. Set `linear.api_key` in `WORKFLOW.md` or export `LINEAR_API_KEY`." } @@ -301,13 +256,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do linear_client: fn _query, _variables, _opts -> {:error, {:linear_api_status, 503}} end ) - assert [ - %{ - "text" => status_error_text - } - ] = status_error["contentItems"] - - assert Jason.decode!(status_error_text) == %{ + assert Jason.decode!(status_error["output"]) == %{ "error" => %{ "message" => "Linear GraphQL request failed with HTTP 503.", "status" => 503 @@ -321,13 +270,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do linear_client: fn _query, _variables, _opts -> {:error, {:linear_api_request, :timeout}} end ) - assert [ - %{ - "text" => request_error_text - } - ] = request_error["contentItems"] - - assert Jason.decode!(request_error_text) == %{ + assert Jason.decode!(request_error["output"]) == %{ "error" => %{ "message" => "Linear GraphQL request failed before receiving a successful response.", "reason" => ":timeout" @@ -345,13 +288,7 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do assert response["success"] == false - assert [ - %{ - "text" => text - } - ] = response["contentItems"] - - assert Jason.decode!(text) == %{ + assert Jason.decode!(response["output"]) == %{ "error" => %{ "message" => "Linear GraphQL tool execution failed.", "reason" => ":boom" @@ -368,11 +305,6 @@ defmodule SymphonyElixir.Codex.DynamicToolTest do ) assert response["success"] == true - - assert [ - %{ - "text" => ":ok" - } - ] = response["contentItems"] + assert response["output"] == ":ok" end end diff --git a/elixir/test/symphony_elixir/extensions_test.exs b/elixir/test/symphony_elixir/extensions_test.exs index 27b2416b7b..d6309c9662 100644 --- a/elixir/test/symphony_elixir/extensions_test.exs +++ b/elixir/test/symphony_elixir/extensions_test.exs @@ -348,6 +348,8 @@ defmodule SymphonyElixir.ExtensionsTest do "issue_id" => "issue-http", "issue_identifier" => "MT-HTTP", "state" => "In Progress", + "worker_host" => nil, + "workspace_path" => nil, "session_id" => "thread-http", "turn_count" => 7, "last_event" => "notification", @@ -363,7 +365,9 @@ defmodule SymphonyElixir.ExtensionsTest do "issue_identifier" => "MT-RETRY", "attempt" => 2, "due_at" => state_payload["retrying"] |> List.first() |> Map.fetch!("due_at"), - "error" => "boom" + "error" => "boom", + "worker_host" => nil, + "workspace_path" => nil } ], "codex_totals" => %{ @@ -382,9 +386,14 @@ defmodule SymphonyElixir.ExtensionsTest do "issue_identifier" => "MT-HTTP", "issue_id" => "issue-http", "status" => "running", - "workspace" => %{"path" => Path.join(Config.settings!().workspace.root, "MT-HTTP")}, + "workspace" => %{ + "path" => Path.join(Config.settings!().workspace.root, "MT-HTTP"), + "host" => nil + }, "attempts" => %{"restart_count" => 0, "current_retry_attempt" => 0}, "running" => %{ + "worker_host" => nil, + "workspace_path" => nil, "session_id" => "thread-http", "turn_count" => 7, "state" => "In Progress", diff --git a/elixir/test/symphony_elixir/live_e2e_test.exs b/elixir/test/symphony_elixir/live_e2e_test.exs index b775e66fcc..9bfeced8c5 100644 --- a/elixir/test/symphony_elixir/live_e2e_test.exs +++ b/elixir/test/symphony_elixir/live_e2e_test.exs @@ -2,25 +2,20 @@ defmodule SymphonyElixir.LiveE2ETest do use SymphonyElixir.TestSupport require Logger + alias SymphonyElixir.SSH @moduletag :live_e2e @moduletag timeout: 300_000 @default_team_key "SYME2E" + @default_docker_auth_json Path.join(System.user_home!(), ".codex/auth.json") + @docker_worker_count 2 + @docker_support_dir Path.expand("../support/live_e2e_docker", __DIR__) + @docker_compose_file Path.join(@docker_support_dir, "docker-compose.yml") @result_file "LIVE_E2E_RESULT.txt" - @live_e2e_skip_reason (cond do - System.get_env("SYMPHONY_RUN_LIVE_E2E") != "1" -> - "set SYMPHONY_RUN_LIVE_E2E=1 to enable the real Linear/Codex end-to-end test" - - is_nil(System.find_executable("codex")) -> - "real Codex live test requires `codex` on PATH" - - System.get_env("LINEAR_API_KEY") in [nil, ""] -> - "real Linear live test requires LINEAR_API_KEY" - - true -> - nil - end) + @live_e2e_skip_reason if(System.get_env("SYMPHONY_RUN_LIVE_E2E") != "1", + do: "set SYMPHONY_RUN_LIVE_E2E=1 to enable the real Linear/Codex end-to-end test" + ) @team_query """ query SymphonyLiveE2ETeam($key: String!) { @@ -126,82 +121,13 @@ defmodule SymphonyElixir.LiveE2ETest do """ @tag skip: @live_e2e_skip_reason - test "creates a real Linear project and issue, then runs a real Codex turn" do - test_root = - Path.join( - System.tmp_dir!(), - "symphony-live-e2e-#{System.unique_integer([:positive])}" - ) - - workflow_root = Path.join(test_root, "workflow") - workflow_file = Path.join(workflow_root, "WORKFLOW.md") - workspace_root = Path.join(test_root, "workspaces") - team_key = System.get_env("SYMPHONY_LIVE_LINEAR_TEAM_KEY") || @default_team_key - codex_command = System.get_env("SYMPHONY_LIVE_CODEX_COMMAND") || "codex app-server" - original_workflow_path = Workflow.workflow_file_path() - - File.mkdir_p!(workflow_root) - - try do - Workflow.set_workflow_file_path(workflow_file) - - write_workflow_file!(workflow_file, - tracker_api_token: "$LINEAR_API_KEY", - tracker_project_slug: "bootstrap", - workspace_root: workspace_root, - codex_command: codex_command, - codex_approval_policy: "never", - observability_enabled: false - ) - - team = fetch_team!(team_key) - active_state = active_state!(team) - completed_project_status = completed_project_status!() - terminal_states = terminal_state_names(team) - - project = - create_project!( - team["id"], - "Symphony Live E2E #{System.unique_integer([:positive])}" - ) - - issue = - create_issue!( - team["id"], - project["id"], - active_state["id"], - "Symphony live e2e issue for #{project["name"]}" - ) - - write_workflow_file!(workflow_file, - tracker_api_token: "$LINEAR_API_KEY", - tracker_project_slug: project["slugId"], - tracker_active_states: [active_state["name"]], - tracker_terminal_states: terminal_states, - workspace_root: workspace_root, - codex_command: codex_command, - codex_approval_policy: "never", - codex_turn_timeout_ms: 600_000, - codex_stall_timeout_ms: 600_000, - observability_enabled: false, - prompt: live_prompt(project["slugId"]) - ) - - assert :ok = AgentRunner.run(issue, nil, max_turns: 1) - - result_path = Path.join([workspace_root, issue.identifier, @result_file]) - assert File.exists?(result_path) - assert File.read!(result_path) == expected_result(issue.identifier, project["slugId"]) - - issue_snapshot = fetch_issue_details!(issue.id) - assert issue_completed?(issue_snapshot) - assert issue_has_comment?(issue_snapshot, expected_comment(issue.identifier, project["slugId"])) + test "creates a real Linear project and issue with a local worker" do + run_live_issue_flow!(:local) + end - assert :ok = complete_project(project["id"], completed_project_status["id"]) - after - Workflow.set_workflow_file_path(original_workflow_path) - File.rm_rf(test_root) - end + @tag skip: @live_e2e_skip_reason + test "creates a real Linear project and issue with an ssh worker" do + run_live_issue_flow!(:ssh) end defp fetch_team!(team_key) do @@ -234,6 +160,16 @@ defmodule SymphonyElixir.LiveE2ETest do end end + defp active_state_names(%{"states" => %{"nodes" => states}}) when is_list(states) do + states + |> Enum.reject(&(&1["type"] in ["completed", "canceled"])) + |> Enum.map(& &1["name"]) + |> case do + [] -> ["Todo", "In Progress", "In Review"] + names -> names + end + end + defp completed_project_status! do @project_statuses_query |> graphql_data!(%{}) @@ -387,10 +323,12 @@ defmodule SymphonyElixir.LiveE2ETest do project_slug=#{project_slug} Step 2: - Use the `linear_graphql` tool to query the current issue by `{{ issue.id }}` and read: + You must use the `linear_graphql` tool to query the current issue by `{{ issue.id }}` and read: - existing comments - team workflow states + A turn that only creates the file is incomplete. Do not stop after Step 1. + If the exact comment body below is not already present, post exactly one comment on the current issue with this exact body: #{expected_comment("{{ issue.identifier }}", project_slug)} @@ -457,4 +395,408 @@ defmodule SymphonyElixir.LiveE2ETest do defp expected_comment(issue_identifier, project_slug) do "Symphony live e2e comment\nidentifier=#{issue_identifier}\nproject_slug=#{project_slug}" end + + defp receive_runtime_info!(issue_id) do + receive do + {:worker_runtime_info, ^issue_id, %{workspace_path: workspace_path} = runtime_info} + when is_binary(workspace_path) -> + runtime_info + + {:codex_worker_update, ^issue_id, _message} -> + receive_runtime_info!(issue_id) + after + 5_000 -> + flunk("timed out waiting for worker runtime info for #{inspect(issue_id)}") + end + end + + defp read_worker_result!(%{worker_host: nil, workspace_path: workspace_path}, result_file) + when is_binary(workspace_path) and is_binary(result_file) do + File.read!(Path.join(workspace_path, result_file)) + end + + defp read_worker_result!(%{worker_host: worker_host, workspace_path: workspace_path}, result_file) + when is_binary(worker_host) and is_binary(workspace_path) and is_binary(result_file) do + remote_result_path = Path.join(workspace_path, result_file) + + case SSH.run(worker_host, "cat #{shell_escape(remote_result_path)}", stderr_to_stdout: true) do + {:ok, {output, 0}} -> + output + + {:ok, {output, status}} -> + flunk("failed to read remote result from #{worker_host}:#{remote_result_path} (status #{status}): #{inspect(output)}") + + {:error, reason} -> + flunk("failed to read remote result from #{worker_host}:#{remote_result_path}: #{inspect(reason)}") + end + end + + defp shell_escape(value) when is_binary(value) do + "'" <> String.replace(value, "'", "'\"'\"'") <> "'" + end + + defp run_live_issue_flow!(backend) when backend in [:local, :ssh] do + run_id = "symphony-live-e2e-#{backend}-#{System.unique_integer([:positive])}" + test_root = Path.join(System.tmp_dir!(), run_id) + workflow_root = Path.join(test_root, "workflow") + workflow_file = Path.join(workflow_root, "WORKFLOW.md") + worker_setup = live_worker_setup!(backend, run_id, test_root) + team_key = System.get_env("SYMPHONY_LIVE_LINEAR_TEAM_KEY") || @default_team_key + original_workflow_path = Workflow.workflow_file_path() + orchestrator_pid = Process.whereis(SymphonyElixir.Orchestrator) + + File.mkdir_p!(workflow_root) + + try do + if is_pid(orchestrator_pid) do + assert :ok = Supervisor.terminate_child(SymphonyElixir.Supervisor, SymphonyElixir.Orchestrator) + end + + Workflow.set_workflow_file_path(workflow_file) + + write_workflow_file!(workflow_file, + tracker_api_token: "$LINEAR_API_KEY", + tracker_project_slug: "bootstrap", + workspace_root: worker_setup.workspace_root, + worker_ssh_hosts: worker_setup.ssh_worker_hosts, + codex_command: worker_setup.codex_command, + codex_approval_policy: "never", + observability_enabled: false + ) + + team = fetch_team!(team_key) + active_state = active_state!(team) + completed_project_status = completed_project_status!() + terminal_states = terminal_state_names(team) + + project = + create_project!( + team["id"], + "Symphony Live E2E #{backend} #{System.unique_integer([:positive])}" + ) + + issue = + create_issue!( + team["id"], + project["id"], + active_state["id"], + "Symphony live e2e #{backend} issue for #{project["name"]}" + ) + + write_workflow_file!(workflow_file, + tracker_api_token: "$LINEAR_API_KEY", + tracker_project_slug: project["slugId"], + tracker_active_states: active_state_names(team), + tracker_terminal_states: terminal_states, + workspace_root: worker_setup.workspace_root, + worker_ssh_hosts: worker_setup.ssh_worker_hosts, + codex_command: worker_setup.codex_command, + codex_approval_policy: "never", + codex_turn_timeout_ms: 600_000, + codex_stall_timeout_ms: 600_000, + observability_enabled: false, + prompt: live_prompt(project["slugId"]) + ) + + assert :ok = AgentRunner.run(issue, self(), max_turns: 3) + + runtime_info = receive_runtime_info!(issue.id) + + assert read_worker_result!(runtime_info, @result_file) == + expected_result(issue.identifier, project["slugId"]) + + issue_snapshot = fetch_issue_details!(issue.id) + assert issue_completed?(issue_snapshot) + assert issue_has_comment?(issue_snapshot, expected_comment(issue.identifier, project["slugId"])) + + assert :ok = complete_project(project["id"], completed_project_status["id"]) + after + restart_orchestrator_if_needed() + cleanup_live_worker_setup(worker_setup) + Workflow.set_workflow_file_path(original_workflow_path) + File.rm_rf(test_root) + end + end + + defp live_worker_setup!(:local, _run_id, test_root) when is_binary(test_root) do + %{ + cleanup: fn -> :ok end, + codex_command: "codex app-server", + ssh_worker_hosts: [], + workspace_root: Path.join(test_root, "workspaces") + } + end + + defp live_worker_setup!(:ssh, run_id, test_root) when is_binary(run_id) and is_binary(test_root) do + case live_ssh_worker_hosts() do + [] -> + live_docker_worker_setup!(run_id, test_root) + + _hosts -> + live_ssh_worker_setup!(run_id) + end + end + + defp cleanup_live_worker_setup(%{cleanup: cleanup}) when is_function(cleanup, 0) do + cleanup.() + end + + defp cleanup_live_worker_setup(_worker_setup), do: :ok + + defp restart_orchestrator_if_needed do + if is_nil(Process.whereis(SymphonyElixir.Orchestrator)) do + case Supervisor.restart_child(SymphonyElixir.Supervisor, SymphonyElixir.Orchestrator) do + {:ok, _pid} -> :ok + {:error, {:already_started, _pid}} -> :ok + end + end + end + + defp live_ssh_worker_setup!(run_id) when is_binary(run_id) do + ssh_worker_hosts = live_ssh_worker_hosts() + remote_test_root = Path.join(shared_remote_home!(ssh_worker_hosts), ".#{run_id}") + remote_workspace_root = "~/.#{run_id}/workspaces" + + %{ + cleanup: fn -> cleanup_remote_test_root(remote_test_root, ssh_worker_hosts) end, + codex_command: "codex app-server", + ssh_worker_hosts: ssh_worker_hosts, + workspace_root: remote_workspace_root + } + end + + defp live_docker_worker_setup!(run_id, test_root) when is_binary(run_id) and is_binary(test_root) do + ssh_root = Path.join(test_root, "live-docker-ssh") + key_path = Path.join(ssh_root, "id_ed25519") + config_path = Path.join(ssh_root, "config") + auth_json_path = @default_docker_auth_json + worker_ports = reserve_tcp_ports(@docker_worker_count) + worker_hosts = Enum.map(worker_ports, &"localhost:#{&1}") + project_name = docker_project_name(run_id) + previous_ssh_config = System.get_env("SYMPHONY_SSH_CONFIG") + + base_cleanup = fn -> + restore_env("SYMPHONY_SSH_CONFIG", previous_ssh_config) + docker_compose_down(project_name, docker_compose_env(worker_ports, auth_json_path, key_path <> ".pub")) + end + + result = + try do + File.mkdir_p!(ssh_root) + generate_ssh_keypair!(key_path) + write_docker_ssh_config!(config_path, key_path) + System.put_env("SYMPHONY_SSH_CONFIG", config_path) + + docker_compose_up!(project_name, docker_compose_env(worker_ports, auth_json_path, key_path <> ".pub")) + wait_for_ssh_hosts!(worker_hosts) + remote_test_root = Path.join(shared_remote_home!(worker_hosts), ".#{run_id}") + remote_workspace_root = "~/.#{run_id}/workspaces" + + %{ + cleanup: fn -> + cleanup_remote_test_root(remote_test_root, worker_hosts) + base_cleanup.() + end, + codex_command: "codex app-server", + ssh_worker_hosts: worker_hosts, + workspace_root: remote_workspace_root + } + rescue + error -> + {:error, error, __STACKTRACE__} + catch + kind, reason -> + {:caught, kind, reason, __STACKTRACE__} + end + + case result do + %{ssh_worker_hosts: _hosts} = worker_setup -> + worker_setup + + {:error, error, stacktrace} -> + base_cleanup.() + reraise(error, stacktrace) + + {:caught, kind, reason, stacktrace} -> + base_cleanup.() + :erlang.raise(kind, reason, stacktrace) + end + end + + defp live_ssh_worker_hosts do + System.get_env("SYMPHONY_LIVE_SSH_WORKER_HOSTS", "") + |> String.split(",", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + end + + defp cleanup_remote_test_root(test_root, ssh_worker_hosts) + when is_binary(test_root) and is_list(ssh_worker_hosts) do + Enum.each(ssh_worker_hosts, fn worker_host -> + _ = SSH.run(worker_host, "rm -rf #{shell_escape(test_root)}", stderr_to_stdout: true) + end) + end + + defp shared_remote_home!([first_host | rest] = worker_hosts) when is_binary(first_host) and rest != [] do + homes = + worker_hosts + |> Enum.map(fn worker_host -> {worker_host, remote_home!(worker_host)} end) + + [{_host, home} | _remaining] = homes + + if Enum.all?(homes, fn {_host, other_home} -> other_home == home end) do + home + else + flunk("expected all live SSH workers to share one home directory, got: #{inspect(homes)}") + end + end + + defp shared_remote_home!([worker_host]) when is_binary(worker_host), do: remote_home!(worker_host) + defp shared_remote_home!(_worker_hosts), do: flunk("expected at least one live SSH worker host") + + defp remote_home!(worker_host) when is_binary(worker_host) do + case SSH.run(worker_host, "printf '%s\\n' \"$HOME\"", stderr_to_stdout: true) do + {:ok, {output, 0}} -> + output + |> String.trim() + |> case do + "" -> flunk("expected non-empty remote home for #{worker_host}") + home -> home + end + + {:ok, {output, status}} -> + flunk("failed to resolve remote home for #{worker_host} (status #{status}): #{inspect(output)}") + + {:error, reason} -> + flunk("failed to resolve remote home for #{worker_host}: #{inspect(reason)}") + end + end + + defp reserve_tcp_ports(count) when is_integer(count) and count > 0 do + reserve_tcp_ports(count, MapSet.new(), []) + end + + defp reserve_tcp_ports(0, _seen, ports), do: Enum.reverse(ports) + + defp reserve_tcp_ports(remaining, seen, ports) do + port = reserve_tcp_port!() + + if MapSet.member?(seen, port) do + reserve_tcp_ports(remaining, seen, ports) + else + reserve_tcp_ports(remaining - 1, MapSet.put(seen, port), [port | ports]) + end + end + + defp reserve_tcp_port! do + {:ok, socket} = :gen_tcp.listen(0, [:binary, {:active, false}, {:reuseaddr, true}]) + {:ok, port} = :inet.port(socket) + :ok = :gen_tcp.close(socket) + port + end + + defp generate_ssh_keypair!(key_path) when is_binary(key_path) do + case System.find_executable("ssh-keygen") do + nil -> + flunk("docker worker mode requires `ssh-keygen` on PATH") + + executable -> + key_dir = Path.dirname(key_path) + File.mkdir_p!(key_dir) + File.rm_rf(key_path) + File.rm_rf(key_path <> ".pub") + + case System.cmd(executable, ["-q", "-t", "ed25519", "-N", "", "-f", key_path], stderr_to_stdout: true) do + {_output, 0} -> :ok + {output, status} -> flunk("failed to generate live docker ssh key (status #{status}): #{inspect(output)}") + end + end + end + + defp write_docker_ssh_config!(config_path, key_path) + when is_binary(config_path) and is_binary(key_path) do + config_contents = """ + Host localhost 127.0.0.1 + User root + IdentityFile #{key_path} + IdentitiesOnly yes + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + LogLevel ERROR + """ + + File.mkdir_p!(Path.dirname(config_path)) + File.write!(config_path, config_contents) + end + + defp docker_project_name(run_id) when is_binary(run_id) do + run_id + |> String.downcase() + |> String.replace(~r/[^a-z0-9_-]/, "-") + end + + defp docker_compose_env(worker_ports, auth_json_path, authorized_key_path) + when is_list(worker_ports) and is_binary(auth_json_path) and is_binary(authorized_key_path) do + [ + {"SYMPHONY_LIVE_DOCKER_AUTH_JSON", auth_json_path}, + {"SYMPHONY_LIVE_DOCKER_AUTHORIZED_KEY", authorized_key_path}, + {"SYMPHONY_LIVE_DOCKER_WORKER_1_PORT", Integer.to_string(Enum.at(worker_ports, 0))}, + {"SYMPHONY_LIVE_DOCKER_WORKER_2_PORT", Integer.to_string(Enum.at(worker_ports, 1))} + ] + end + + defp docker_compose_up!(project_name, env) when is_binary(project_name) and is_list(env) do + args = ["compose", "-f", @docker_compose_file, "-p", project_name, "up", "-d", "--build"] + + case System.cmd("docker", args, cd: @docker_support_dir, env: env, stderr_to_stdout: true) do + {_output, 0} -> + :ok + + {output, status} -> + flunk("failed to start live docker workers (status #{status}): #{inspect(output)}") + end + end + + defp docker_compose_down(project_name, env) when is_binary(project_name) and is_list(env) do + _ = + System.cmd( + "docker", + ["compose", "-f", @docker_compose_file, "-p", project_name, "down", "-v", "--remove-orphans"], + cd: @docker_support_dir, + env: env, + stderr_to_stdout: true + ) + + :ok + end + + defp wait_for_ssh_hosts!(worker_hosts) when is_list(worker_hosts) do + deadline = System.monotonic_time(:millisecond) + 60_000 + + Enum.each(worker_hosts, fn worker_host -> + wait_for_ssh_host!(worker_host, deadline) + end) + end + + defp wait_for_ssh_host!(worker_host, deadline_ms) when is_binary(worker_host) do + case SSH.run(worker_host, "printf ready", stderr_to_stdout: true) do + {:ok, {"ready", 0}} -> + :ok + + {:ok, {_output, _status}} -> + retry_or_flunk_ssh_host(worker_host, deadline_ms) + + {:error, _reason} -> + retry_or_flunk_ssh_host(worker_host, deadline_ms) + end + end + + defp retry_or_flunk_ssh_host(worker_host, deadline_ms) do + if System.monotonic_time(:millisecond) < deadline_ms do + Process.sleep(1_000) + wait_for_ssh_host!(worker_host, deadline_ms) + else + flunk("timed out waiting for SSH worker #{worker_host} to accept connections") + end + end end diff --git a/elixir/test/symphony_elixir/orchestrator_status_test.exs b/elixir/test/symphony_elixir/orchestrator_status_test.exs index 14c3e1bb28..4326b80ce3 100644 --- a/elixir/test/symphony_elixir/orchestrator_status_test.exs +++ b/elixir/test/symphony_elixir/orchestrator_status_test.exs @@ -767,6 +767,8 @@ defmodule SymphonyElixir.OrchestratorStatusTest do %{ state | poll_interval_ms: 30_000, + tick_timer_ref: nil, + tick_token: make_ref(), next_poll_due_at_ms: now_ms + 4_000, poll_check_in_progress: false } diff --git a/elixir/test/symphony_elixir/ssh_test.exs b/elixir/test/symphony_elixir/ssh_test.exs new file mode 100644 index 0000000000..9edc94f3af --- /dev/null +++ b/elixir/test/symphony_elixir/ssh_test.exs @@ -0,0 +1,199 @@ +defmodule SymphonyElixir.SSHTest do + use ExUnit.Case, async: false + + alias SymphonyElixir.SSH + + test "run/3 keeps bracketed IPv6 host:port targets intact" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-ipv6-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + + on_exit(fn -> + restore_env("PATH", previous_path) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file) + + assert {:ok, {"", 0}} = + SSH.run("root@[::1]:2200", "printf ok", stderr_to_stdout: true) + + trace = File.read!(trace_file) + assert trace =~ "-T -p 2200 root@[::1] bash -lc" + assert trace =~ "printf ok" + end + + test "run/3 leaves unbracketed IPv6-style targets unchanged" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-ipv6-raw-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + + on_exit(fn -> + restore_env("PATH", previous_path) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file) + + assert {:ok, {"", 0}} = + SSH.run("::1:2200", "printf ok", stderr_to_stdout: true) + + trace = File.read!(trace_file) + assert trace =~ "-T ::1:2200 bash -lc" + refute trace =~ "-p 2200" + end + + test "run/3 passes host:port targets through ssh -p" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + previous_ssh_config = System.get_env("SYMPHONY_SSH_CONFIG") + + on_exit(fn -> + restore_env("PATH", previous_path) + restore_env("SYMPHONY_SSH_CONFIG", previous_ssh_config) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file) + System.put_env("SYMPHONY_SSH_CONFIG", "/tmp/symphony-test-ssh-config") + + assert {:ok, {"", 0}} = + SSH.run("localhost:2222", "echo ready", stderr_to_stdout: true) + + trace = File.read!(trace_file) + assert trace =~ "-F /tmp/symphony-test-ssh-config" + assert trace =~ "-T -p 2222 localhost bash -lc" + assert trace =~ "echo ready" + end + + test "run/3 keeps the user prefix when parsing user@host:port targets" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-user-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + + on_exit(fn -> + restore_env("PATH", previous_path) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file) + + assert {:ok, {"", 0}} = + SSH.run("root@127.0.0.1:2200", "printf ok", stderr_to_stdout: true) + + trace = File.read!(trace_file) + assert trace =~ "-T -p 2200 root@127.0.0.1 bash -lc" + assert trace =~ "printf ok" + end + + test "run/3 returns an error when ssh is unavailable" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-missing-test-#{System.unique_integer([:positive])}") + previous_path = System.get_env("PATH") + + on_exit(fn -> + restore_env("PATH", previous_path) + File.rm_rf(test_root) + end) + + File.mkdir_p!(test_root) + System.put_env("PATH", test_root) + + assert {:error, :ssh_not_found} = SSH.run("localhost", "printf ok") + end + + test "start_port/3 supports binary output without line mode" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-port-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + previous_ssh_config = System.get_env("SYMPHONY_SSH_CONFIG") + + on_exit(fn -> + restore_env("PATH", previous_path) + restore_env("SYMPHONY_SSH_CONFIG", previous_ssh_config) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file, """ + #!/bin/sh + printf 'ARGV:%s\\n' "$*" >> "#{trace_file}" + printf 'ready\\n' + exit 0 + """) + + System.delete_env("SYMPHONY_SSH_CONFIG") + + assert {:ok, port} = SSH.start_port("localhost", "printf ok") + assert is_port(port) + wait_for_trace!(trace_file) + + trace = File.read!(trace_file) + assert trace =~ "-T localhost bash -lc" + refute trace =~ " -F " + end + + test "start_port/3 supports line mode" do + test_root = Path.join(System.tmp_dir!(), "symphony-ssh-line-port-test-#{System.unique_integer([:positive])}") + trace_file = Path.join(test_root, "ssh.trace") + previous_path = System.get_env("PATH") + + on_exit(fn -> + restore_env("PATH", previous_path) + File.rm_rf(test_root) + end) + + install_fake_ssh!(test_root, trace_file, """ + #!/bin/sh + printf 'ARGV:%s\\n' "$*" >> "#{trace_file}" + printf 'ready\\n' + exit 0 + """) + + assert {:ok, port} = SSH.start_port("localhost:2222", "printf ok", line: 256) + assert is_port(port) + wait_for_trace!(trace_file) + + trace = File.read!(trace_file) + assert trace =~ "-T -p 2222 localhost bash -lc" + end + + test "remote_shell_command/1 escapes embedded single quotes" do + assert SSH.remote_shell_command("printf 'hello'") == + "bash -lc 'printf '\"'\"'hello'\"'\"''" + end + + defp install_fake_ssh!(test_root, trace_file, script \\ nil) do + fake_bin_dir = Path.join(test_root, "bin") + fake_ssh = Path.join(fake_bin_dir, "ssh") + + File.mkdir_p!(fake_bin_dir) + + File.write!( + fake_ssh, + script || + """ + #!/bin/sh + printf 'ARGV:%s\\n' "$*" >> "#{trace_file}" + exit 0 + """ + ) + + File.chmod!(fake_ssh, 0o755) + System.put_env("PATH", fake_bin_dir <> ":" <> (System.get_env("PATH") || "")) + end + + defp wait_for_trace!(trace_file, attempts \\ 20) + defp wait_for_trace!(trace_file, 0), do: flunk("timed out waiting for fake ssh trace at #{trace_file}") + + defp wait_for_trace!(trace_file, attempts) do + if File.exists?(trace_file) and File.read!(trace_file) != "" do + :ok + else + Process.sleep(25) + wait_for_trace!(trace_file, attempts - 1) + end + end + + defp restore_env(key, nil), do: System.delete_env(key) + defp restore_env(key, value), do: System.put_env(key, value) +end diff --git a/elixir/test/symphony_elixir/workspace_and_config_test.exs b/elixir/test/symphony_elixir/workspace_and_config_test.exs index 03a3c59ee4..59ff0850b1 100644 --- a/elixir/test/symphony_elixir/workspace_and_config_test.exs +++ b/elixir/test/symphony_elixir/workspace_and_config_test.exs @@ -86,7 +86,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert File.read!(Path.join(second_workspace, "local-progress.txt")) == "in progress\n" assert File.read!(Path.join([second_workspace, "deps", "cache.txt"])) == "cached deps\n" assert File.read!(Path.join([second_workspace, "_build", "artifact.txt"])) == "compiled artifact\n" - refute File.exists?(Path.join([second_workspace, "tmp", "scratch.txt"])) + assert File.read!(Path.join([second_workspace, "tmp", "scratch.txt"])) == "remove me\n" after File.rm_rf(workspace_root) end @@ -742,6 +742,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert config.tracker.api_key == nil assert config.tracker.project_slug == nil assert config.workspace.root == Path.join(System.tmp_dir!(), "symphony_workspaces") + assert config.worker.max_concurrent_agents_per_host == nil assert config.agent.max_concurrent_agents == 10 assert config.codex.command == "codex app-server" @@ -813,6 +814,10 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert {:error, {:invalid_workflow_config, message}} = Config.validate!() assert message =~ "agent.max_concurrent_agents" + write_workflow_file!(Workflow.workflow_file_path(), worker_max_concurrent_agents_per_host: 0) + assert {:error, {:invalid_workflow_config, message}} = Config.validate!() + assert message =~ "worker.max_concurrent_agents_per_host" + write_workflow_file!(Workflow.workflow_file_path(), codex_turn_timeout_ms: "bad") assert {:error, {:invalid_workflow_config, message}} = Config.validate!() assert message =~ "codex.turn_timeout_ms" @@ -932,7 +937,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do config = Config.settings!() assert config.tracker.api_key == "env:#{api_key_env_var}" - assert config.workspace.root == Path.expand("env:#{workspace_env_var}") + assert config.workspace.root == "env:#{workspace_env_var}" end test "config supports per-state max concurrent agent overrides" do @@ -955,6 +960,10 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert Config.max_concurrent_agents_for_state("In Review") == 2 assert Config.max_concurrent_agents_for_state("Closed") == 10 assert Config.max_concurrent_agents_for_state(:not_a_string) == 10 + + write_workflow_file!(Workflow.workflow_file_path(), worker_max_concurrent_agents_per_host: 2) + assert :ok = Config.validate!() + assert Config.settings!().worker.max_concurrent_agents_per_host == 2 end test "schema helpers cover custom type and state limit validation" do @@ -1021,7 +1030,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do }) assert settings.tracker.api_key == nil - assert settings.workspace.root == Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + assert settings.workspace.root == Path.join(System.tmp_dir!(), "symphony_workspaces") assert settings.codex.approval_policy == %{ "reject" => %{"sandbox_approval" => true} @@ -1034,7 +1043,7 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do }) assert settings.tracker.api_key == "fallback-linear-token" - assert settings.workspace.root == Path.expand(Path.join(System.tmp_dir!(), "symphony_workspaces")) + assert settings.workspace.root == Path.join(System.tmp_dir!(), "symphony_workspaces") end test "schema resolves sandbox policies from explicit and default workspaces" do @@ -1073,6 +1082,37 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do } end + test "schema keeps workspace roots raw while sandbox helpers expand only for local use" do + assert {:ok, settings} = + Schema.parse(%{ + workspace: %{root: "~/.symphony-workspaces"}, + codex: %{} + }) + + assert settings.workspace.root == "~/.symphony-workspaces" + + assert Schema.resolve_turn_sandbox_policy(settings) == %{ + "type" => "workspaceWrite", + "writableRoots" => [Path.expand("~/.symphony-workspaces")], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + + assert {:ok, remote_policy} = + Schema.resolve_runtime_turn_sandbox_policy(settings, nil, remote: true) + + assert remote_policy == %{ + "type" => "workspaceWrite", + "writableRoots" => ["~/.symphony-workspaces"], + "readOnlyAccess" => %{"type" => "fullAccess"}, + "networkAccess" => false, + "excludeTmpdirEnvVar" => false, + "excludeSlashTmp" => false + } + end + test "runtime sandbox policy resolution passes explicit policies through unchanged" do test_root = Path.join( @@ -1154,6 +1194,11 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert default_policy["type"] == "workspaceWrite" assert default_policy["writableRoots"] == [canonical_workspace_root] + assert {:ok, blank_workspace_policy} = + Schema.resolve_runtime_turn_sandbox_policy(settings, "") + + assert blank_workspace_policy == default_policy + read_only_settings = %{ settings | codex: %{settings.codex | turn_sandbox_policy: %{"type" => "readOnly", "networkAccess" => true}} @@ -1183,4 +1228,75 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do write_workflow_file!(Workflow.workflow_file_path(), prompt: workflow_prompt) assert Config.workflow_prompt() == workflow_prompt end + + test "remote workspace lifecycle uses ssh host aliases from worker config" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-remote-workspace-#{System.unique_integer([:positive])}" + ) + + previous_path = System.get_env("PATH") + previous_trace = System.get_env("SYMP_TEST_SSH_TRACE") + + on_exit(fn -> + restore_env("PATH", previous_path) + restore_env("SYMP_TEST_SSH_TRACE", previous_trace) + end) + + try do + trace_file = Path.join(test_root, "ssh.trace") + fake_ssh = Path.join(test_root, "ssh") + workspace_root = "~/.symphony-remote-workspaces" + workspace_path = "/remote/home/.symphony-remote-workspaces/MT-SSH-WS" + + File.mkdir_p!(test_root) + System.put_env("SYMP_TEST_SSH_TRACE", trace_file) + System.put_env("PATH", test_root <> ":" <> (previous_path || "")) + + File.write!(fake_ssh, """ + #!/bin/sh + trace_file="${SYMP_TEST_SSH_TRACE:-/tmp/symphony-fake-ssh.trace}" + printf 'ARGV:%s\\n' "$*" >> "$trace_file" + + case "$*" in + *"__SYMPHONY_WORKSPACE__"*) + printf '%s\\t%s\\t%s\\n' '__SYMPHONY_WORKSPACE__' '1' '#{workspace_path}' + ;; + esac + + exit 0 + """) + + File.chmod!(fake_ssh, 0o755) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + worker_ssh_hosts: ["worker-01:2200"], + hook_before_run: "echo before-run", + hook_after_run: "echo after-run", + hook_before_remove: "echo before-remove" + ) + + assert Config.settings!().worker.ssh_hosts == ["worker-01:2200"] + assert Config.settings!().workspace.root == workspace_root + assert {:ok, ^workspace_path} = Workspace.create_for_issue("MT-SSH-WS", "worker-01:2200") + assert :ok = Workspace.run_before_run_hook(workspace_path, "MT-SSH-WS", "worker-01:2200") + assert :ok = Workspace.run_after_run_hook(workspace_path, "MT-SSH-WS", "worker-01:2200") + assert :ok = Workspace.remove_issue_workspaces("MT-SSH-WS", "worker-01:2200") + + trace = File.read!(trace_file) + assert trace =~ "-p 2200 worker-01 bash -lc" + assert trace =~ "__SYMPHONY_WORKSPACE__" + assert trace =~ "~/.symphony-remote-workspaces/MT-SSH-WS" + assert trace =~ "${workspace#~/}" + assert trace =~ "echo before-run" + assert trace =~ "echo after-run" + assert trace =~ "echo before-remove" + assert trace =~ "rm -rf" + assert trace =~ workspace_path + after + File.rm_rf(test_root) + end + end end From 1f86bac53a84eb0e9f10d6546e3f19a5724a5b09 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Mar 2026 10:09:10 -0700 Subject: [PATCH 06/41] fix(elixir): fix malformed JSON event form codex message (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Context The Codex app-server merges stderr into stdout (`stderr_to_stdout`). Non-JSON diagnostic lines written by Codex during healthy turns were incorrectly emitted as `:malformed` protocol events in the Symphony UI. Screenshot 2026-03-12 at 4 27 20 PM #### TL;DR *Only emit `:malformed` events for JSON-like protocol frames (lines starting with `{`); silently log all other non-JSON stream output.* #### Summary - Gate `:malformed` event emission on a new `protocol_message_candidate?/1` check — only lines starting with `{` - Preserve full logging of all non-JSON stderr noise via `log_non_json_stream_line` - Fix existing "Capture stderr log" test to assert no `:malformed` event is emitted for stderr noise - Add regression test verifying truncated JSON-like frames still surface as `:malformed` #### Alternatives - Separate stderr from stdout with a dedicated pipe: more invasive change, unnecessary given the simple heuristic - Suppress all non-JSON output silently: would lose valuable diagnostic logging #### Test Plan - [x] `make -C elixir all` - [x] `mise exec -- mix format lib/symphony_elixir/codex/app_server.ex test/symphony_elixir/app_server_test.exs` - [x] Additional: `MIX_ENV=test mise exec -- mix run --no-start -e 'Application.put_env(:symphony_elixir, :workflow_file_path, System.fetch_env!("SYMPHONY_TEST_WORKFLOW")); Mix.Task.run("test", ["test/symphony_elixir/app_server_test.exs"])'` Co-authored-by: Codex --- .../lib/symphony_elixir/codex/app_server.ex | 27 ++++--- .../test/symphony_elixir/app_server_test.exs | 79 ++++++++++++++++++- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index 7da87ce998..9f36bc7a0c 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -422,15 +422,17 @@ defmodule SymphonyElixir.Codex.AppServer do {:error, _reason} -> log_non_json_stream_line(payload_string, "turn stream") - emit_message( - on_message, - :malformed, - %{ - payload: payload_string, - raw: payload_string - }, - metadata_from_message(port, %{raw: payload_string}) - ) + if protocol_message_candidate?(payload_string) do + emit_message( + on_message, + :malformed, + %{ + payload: payload_string, + raw: payload_string + }, + metadata_from_message(port, %{raw: payload_string}) + ) + end receive_loop(port, on_message, timeout_ms, "", tool_executor, auto_approve_requests) end @@ -977,6 +979,13 @@ defmodule SymphonyElixir.Codex.AppServer do end end + defp protocol_message_candidate?(data) do + data + |> to_string() + |> String.trim_leading() + |> String.starts_with?("{") + end + defp issue_context(%{id: issue_id, identifier: identifier}) do "issue_id=#{issue_id} issue_identifier=#{identifier}" end diff --git a/elixir/test/symphony_elixir/app_server_test.exs b/elixir/test/symphony_elixir/app_server_test.exs index b7fab1521c..d03627f8b3 100644 --- a/elixir/test/symphony_elixir/app_server_test.exs +++ b/elixir/test/symphony_elixir/app_server_test.exs @@ -1188,17 +1188,94 @@ defmodule SymphonyElixir.AppServerTest do labels: ["backend"] } + test_pid = self() + on_message = fn message -> send(test_pid, {:app_server_message, message}) end + log = capture_log(fn -> - assert {:ok, _result} = AppServer.run(workspace, "Capture stderr log", issue) + assert {:ok, _result} = + AppServer.run(workspace, "Capture stderr log", issue, on_message: on_message) end) + assert_received {:app_server_message, %{event: :turn_completed}} + refute_received {:app_server_message, %{event: :malformed}} assert log =~ "Codex turn stream output: warning: this is stderr noise" after File.rm_rf(test_root) end end + test "app server emits malformed events for JSON-like protocol lines that fail to decode" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-app-server-malformed-protocol-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + workspace = Path.join(workspace_root, "MT-93") + codex_binary = Path.join(test_root, "fake-codex") + File.mkdir_p!(workspace) + + File.write!(codex_binary, """ + #!/bin/sh + count=0 + while IFS= read -r line; do + count=$((count + 1)) + + case "$count" in + 1) + printf '%s\\n' '{"id":1,"result":{}}' + ;; + 2) + printf '%s\\n' '{"id":2,"result":{"thread":{"id":"thread-93"}}}' + ;; + 3) + printf '%s\\n' '{"id":3,"result":{"turn":{"id":"turn-93"}}}' + ;; + 4) + printf '%s\\n' '{"method":"turn/completed"' + printf '%s\\n' '{"method":"turn/completed"}' + exit 0 + ;; + *) + exit 0 + ;; + esac + done + """) + + File.chmod!(codex_binary, 0o755) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + codex_command: "#{codex_binary} app-server" + ) + + issue = %Issue{ + id: "issue-malformed-protocol", + identifier: "MT-93", + title: "Malformed protocol frame", + description: "Ensure malformed JSON-like frames are surfaced to the orchestrator", + state: "In Progress", + url: "https://example.org/issues/MT-93", + labels: ["backend"] + } + + test_pid = self() + on_message = fn message -> send(test_pid, {:app_server_message, message}) end + + assert {:ok, _result} = + AppServer.run(workspace, "Capture malformed protocol line", issue, on_message: on_message) + + assert_received {:app_server_message, %{event: :malformed, payload: "{\"method\":\"turn/completed\""}} + assert_received {:app_server_message, %{event: :turn_completed}} + after + File.rm_rf(test_root) + end + end + test "app server launches over ssh for remote workers" do test_root = Path.join( From a164593aacb3db4d6808adc5a87173d906726406 Mon Sep 17 00:00:00 2001 From: mstrautmann-oai Date: Thu, 19 Mar 2026 12:34:27 -0700 Subject: [PATCH 07/41] fix(elixir): keep ssh retries in orchestrator (#54) #### Context SSH failover currently happens inside `AgentRunner`, which can bypass per-host caps and rerun a ticket on a second host after the first host already started setup. #### TL;DR *Keep each worker run on one SSH host and let the orchestrator own retries.* #### Summary - Remove `AgentRunner`'s internal cross-host failover loop - Keep a worker lifetime pinned to one selected SSH host - Add a regression test proving startup failure on one host does not fall through to another #### Alternatives - Keep internal failover and classify retryable errors, but that duplicates orchestrator scheduling logic - Only patch per-host cap enforcement, but invisible cross-host reruns would still risk duplicate side effects #### Test Plan - [x] `make -C elixir all` - [x] `cd elixir && mise exec -- mix test test/symphony_elixir/core_test.exs:1167 test/symphony_elixir/core_test.exs:1237 test/symphony_elixir/core_test.exs:1337 --seed 0` --- elixir/lib/symphony_elixir/agent_runner.ex | 43 +++---------- elixir/test/symphony_elixir/core_test.exs | 70 ++++++++++++++++++++++ 2 files changed, 79 insertions(+), 34 deletions(-) diff --git a/elixir/lib/symphony_elixir/agent_runner.ex b/elixir/lib/symphony_elixir/agent_runner.ex index 35ea8a03e9..502c8cf5f6 100644 --- a/elixir/lib/symphony_elixir/agent_runner.ex +++ b/elixir/lib/symphony_elixir/agent_runner.ex @@ -11,12 +11,12 @@ defmodule SymphonyElixir.AgentRunner do @spec run(map(), pid() | nil, keyword()) :: :ok | no_return() def run(issue, codex_update_recipient \\ nil, opts \\ []) do - worker_hosts = - candidate_worker_hosts(Keyword.get(opts, :worker_host), Config.settings!().worker.ssh_hosts) + # The orchestrator owns host retries so one worker lifetime never hops machines. + worker_host = selected_worker_host(Keyword.get(opts, :worker_host), Config.settings!().worker.ssh_hosts) - Logger.info("Starting agent run for #{issue_context(issue)} worker_hosts=#{inspect(worker_hosts_for_log(worker_hosts))}") + Logger.info("Starting agent run for #{issue_context(issue)} worker_host=#{worker_host_for_log(worker_host)}") - case run_on_worker_hosts(issue, codex_update_recipient, opts, worker_hosts) do + case run_on_worker_host(issue, codex_update_recipient, opts, worker_host) do :ok -> :ok @@ -26,22 +26,6 @@ defmodule SymphonyElixir.AgentRunner do end end - defp run_on_worker_hosts(issue, codex_update_recipient, opts, [worker_host | rest]) do - case run_on_worker_host(issue, codex_update_recipient, opts, worker_host) do - :ok -> - :ok - - {:error, reason} when rest != [] -> - Logger.warning("Agent run failed for #{issue_context(issue)} worker_host=#{worker_host_for_log(worker_host)} reason=#{inspect(reason)}; trying next worker host") - run_on_worker_hosts(issue, codex_update_recipient, opts, rest) - - {:error, reason} -> - {:error, reason} - end - end - - defp run_on_worker_hosts(_issue, _codex_update_recipient, _opts, []), do: {:error, :no_worker_hosts_available} - defp run_on_worker_host(issue, codex_update_recipient, opts, worker_host) do Logger.info("Starting worker attempt for #{issue_context(issue)} worker_host=#{worker_host_for_log(worker_host)}") @@ -188,9 +172,9 @@ defmodule SymphonyElixir.AgentRunner do defp active_issue_state?(_state_name), do: false - defp candidate_worker_hosts(nil, []), do: [nil] + defp selected_worker_host(nil, []), do: nil - defp candidate_worker_hosts(preferred_host, configured_hosts) when is_list(configured_hosts) do + defp selected_worker_host(preferred_host, configured_hosts) when is_list(configured_hosts) do hosts = configured_hosts |> Enum.map(&String.trim/1) @@ -198,21 +182,12 @@ defmodule SymphonyElixir.AgentRunner do |> Enum.uniq() case preferred_host do - host when is_binary(host) and host != "" -> - [host | Enum.reject(hosts, &(&1 == host))] - - _ when hosts == [] -> - [nil] - - _ -> - hosts + host when is_binary(host) and host != "" -> host + _ when hosts == [] -> nil + _ -> List.first(hosts) end end - defp worker_hosts_for_log(worker_hosts) do - Enum.map(worker_hosts, &worker_host_for_log/1) - end - defp worker_host_for_log(nil), do: "local" defp worker_host_for_log(worker_host), do: worker_host diff --git a/elixir/test/symphony_elixir/core_test.exs b/elixir/test/symphony_elixir/core_test.exs index 2e33239382..dcbe12a271 100644 --- a/elixir/test/symphony_elixir/core_test.exs +++ b/elixir/test/symphony_elixir/core_test.exs @@ -1164,6 +1164,76 @@ defmodule SymphonyElixir.CoreTest do end end + test "agent runner surfaces ssh startup failures instead of silently hopping hosts" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-agent-runner-single-host-#{System.unique_integer([:positive])}" + ) + + previous_path = System.get_env("PATH") + previous_trace = System.get_env("SYMP_TEST_SSH_TRACE") + + on_exit(fn -> + restore_env("PATH", previous_path) + restore_env("SYMP_TEST_SSH_TRACE", previous_trace) + end) + + try do + trace_file = Path.join(test_root, "ssh.trace") + fake_ssh = Path.join(test_root, "ssh") + + File.mkdir_p!(test_root) + System.put_env("SYMP_TEST_SSH_TRACE", trace_file) + System.put_env("PATH", test_root <> ":" <> (previous_path || "")) + + File.write!(fake_ssh, """ + #!/bin/sh + trace_file="${SYMP_TEST_SSH_TRACE:-/tmp/symphony-fake-ssh.trace}" + printf 'ARGV:%s\\n' "$*" >> "$trace_file" + + case "$*" in + *worker-a*"__SYMPHONY_WORKSPACE__"*) + printf '%s\\n' 'worker-a prepare failed' >&2 + exit 75 + ;; + *worker-b*"__SYMPHONY_WORKSPACE__"*) + printf '%s\\t%s\\t%s\\n' '__SYMPHONY_WORKSPACE__' '1' '/remote/home/.symphony-remote-workspaces/MT-SSH-FAILOVER' + exit 0 + ;; + *) + exit 0 + ;; + esac + """) + + File.chmod!(fake_ssh, 0o755) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: "~/.symphony-remote-workspaces", + worker_ssh_hosts: ["worker-a", "worker-b"] + ) + + issue = %Issue{ + id: "issue-ssh-failover", + identifier: "MT-SSH-FAILOVER", + title: "Do not fail over within a single worker run", + description: "Surface the startup failure to the orchestrator", + state: "In Progress" + } + + assert_raise RuntimeError, ~r/workspace_prepare_failed/, fn -> + AgentRunner.run(issue, nil, worker_host: "worker-a") + end + + trace = File.read!(trace_file) + assert trace =~ "worker-a bash -lc" + refute trace =~ "worker-b bash -lc" + after + File.rm_rf(test_root) + end + end + test "agent runner continues with a follow-up turn while the issue remains active" do test_root = Path.join( From 9e89dd9ff0a3eddb8813c77f633ca4534d6e14b2 Mon Sep 17 00:00:00 2001 From: Drew Hintz Date: Fri, 27 Mar 2026 13:39:50 -0500 Subject: [PATCH 08/41] [codex] Pin GitHub Actions workflow references (#57) ## Summary Pin floating external GitHub Actions workflow refs to immutable SHAs. ## Why See the rationale doc: https://docs.google.com/document/d/1qOURCNx2zszQ0uWx7Fj5ERu4jpiYjxLVWBWgKa2wTsA/edit?tab=t.0 ## Validation - `rg -n --pcre2 "uses:\s*(?!\./)(?!docker://)[^#\n]+@(?![0-9a-f]{40}(?:\s+#.*)?$)\S+" .github/workflows` - `git diff --check` - `git diff --stat -- .github/workflows` --- .github/workflows/make-all.yml | 6 +++--- .github/workflows/pr-description-lint.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/make-all.yml b/.github/workflows/make-all.yml index ca2ab27449..f27fc7c666 100644 --- a/.github/workflows/make-all.yml +++ b/.github/workflows/make-all.yml @@ -15,17 +15,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up mise tools - uses: jdx/mise-action@v3 + uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3 with: install: true cache: true working_directory: elixir - name: Cache deps and build - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: | elixir/deps diff --git a/.github/workflows/pr-description-lint.yml b/.github/workflows/pr-description-lint.yml index 9f2e6ac5fc..2532abfea4 100644 --- a/.github/workflows/pr-description-lint.yml +++ b/.github/workflows/pr-description-lint.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up mise tools - uses: jdx/mise-action@v3 + uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3 with: install: true cache: true From eaa457d96acd59c6081c3d9c86948451dbcd25bf Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Mon, 27 Apr 2026 11:02:32 -0700 Subject: [PATCH 09/41] Clarify Symphony service specification (#61) #### Context SPEC.md had ambiguous conformance language and implementation details that made the service contract harder to port. #### TL;DR *Clarify SPEC.md and normalize RFC-style requirement wording.* #### Summary - Add normative language and tighten config, workspace, reload, restart, and user-input wording. - Move extension config ownership to extension sections. - Point Codex protocol details at the targeted app-server spec instead of duplicating payloads. - Run a careful RFC 2119 terminology pass after mechanical normalization. #### Alternatives - Left attempt/retry redesign and broader safety issues for follow-up spec changes. #### Test Plan - [x] `make -C elixir all` - [x] `git diff --check HEAD~1..HEAD` - [x] RFC keyword scan outside code fences --------- Co-authored-by: Codex --- SPEC.md | 590 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 292 insertions(+), 298 deletions(-) diff --git a/SPEC.md b/SPEC.md index f9e2b63a14..adb8bc59a8 100644 --- a/SPEC.md +++ b/SPEC.md @@ -4,6 +4,15 @@ Status: Draft v1 (language-agnostic) Purpose: Define a service that orchestrates coding agents to get project work done. +## Normative Language + +The key words `MUST`, `MUST NOT`, `REQUIRED`, `SHOULD`, `SHOULD NOT`, `RECOMMENDED`, `MAY`, and +`OPTIONAL` in this document are to be interpreted as described in RFC 2119. + +`Implementation-defined` means the behavior is part of the implementation contract, but this +specification does not prescribe one universal policy. Implementations MUST document the selected +behavior. + ## 1. Problem Statement Symphony is a long-running automation service that continuously reads work from an issue tracker @@ -21,15 +30,15 @@ The service solves four operational problems: Implementations are expected to document their trust and safety posture explicitly. This specification does not require a single approval, sandbox, or operator-confirmation policy; some -implementations may target trusted environments with a high-trust configuration, while others may -require stricter approvals or sandboxing. +implementations target trusted environments with a high-trust configuration, while others require +stricter approvals or sandboxing. Important boundary: - Symphony is a scheduler/runner and tracker reader. - Ticket writes (state transitions, comments, PR links) are typically performed by the coding agent using tools available in the workflow/runtime environment. -- A successful run may end at a workflow-defined handoff state (for example `Human Review`), not +- A successful run can end at a workflow-defined handoff state (for example `Human Review`), not necessarily `Done`. ## 2. Goals and Non-Goals @@ -43,7 +52,8 @@ Important boundary: - Recover from transient failures with exponential backoff. - Load runtime behavior from a repository-owned `WORKFLOW.md` contract. - Expose operator-visible observability (at minimum structured logs). -- Support restart recovery without requiring a persistent database. +- Support tracker/filesystem-driven restart recovery without requiring a persistent database; exact + in-memory scheduler state is not restored. ### 2.2 Non-Goals @@ -94,7 +104,7 @@ Important boundary: - Launches the coding agent app-server client. - Streams agent updates back to the orchestrator. -7. `Status Surface` (optional) +7. `Status Surface` (OPTIONAL) - Presents human-readable runtime status (for example terminal output, dashboard, or other operator-facing view). @@ -122,15 +132,15 @@ Symphony is easiest to port when kept in these layers: 5. `Integration Layer` (Linear adapter) - API calls and normalization for tracker data. -6. `Observability Layer` (logs + optional status surface) +6. `Observability Layer` (logs + OPTIONAL status surface) - Operator visibility into orchestrator and agent behavior. ### 3.3 External Dependencies - Issue tracker API (Linear for `tracker.kind: linear` in this specification version). - Local filesystem for workspaces and logs. -- Optional workspace population tooling (for example Git CLI, if used). -- Coding-agent executable that supports JSON-RPC-like app-server mode over stdio. +- OPTIONAL workspace population tooling (for example Git CLI, if used). +- Coding-agent executable that supports the targeted Codex app-server mode. - Host environment authentication for the issue tracker and coding agent. ## 4. Core Domain Model @@ -194,8 +204,7 @@ Filesystem workspace assigned to one issue identifier. Fields (logical): -- `path` (workspace path; current runtime typically uses absolute paths, but relative roots are - possible if configured without path separators) +- `path` (absolute workspace path) - `workspace_key` (sanitized issue identifier) - `created_now` (boolean, used to gate `after_create` hook) @@ -211,7 +220,7 @@ Fields (logical): - `workspace_path` - `started_at` - `status` -- `error` (optional) +- `error` (OPTIONAL) #### 4.1.6 Live Session (Agent Session Metadata) @@ -293,11 +302,11 @@ Loader behavior: ### 5.2 File Format -`WORKFLOW.md` is a Markdown file with optional YAML front matter. +`WORKFLOW.md` is a Markdown file with OPTIONAL YAML front matter. Design note: -- `WORKFLOW.md` should be self-contained enough to describe and run different workflows (prompt, +- `WORKFLOW.md` SHOULD be self-contained enough to describe and run different workflows (prompt, runtime settings, hooks, and tracker selection/config) without requiring out-of-band service-specific configuration. @@ -306,7 +315,7 @@ Parsing rules: - If file starts with `---`, parse lines until the next `---` as YAML front matter. - Remaining lines become the prompt body. - If front matter is absent, treat the entire file as prompt body and use an empty config map. -- YAML front matter must decode to a map/object; non-map YAML is an error. +- YAML front matter MUST decode to a map/object; non-map YAML is an error. - Prompt body is trimmed before use. Returned workflow object: @@ -325,32 +334,30 @@ Top-level keys: - `agent` - `codex` -Unknown keys should be ignored for forward compatibility. +Unknown keys SHOULD be ignored for forward compatibility. Note: -- The workflow front matter is extensible. Optional extensions may define additional top-level keys - (for example `server`) without changing the core schema above. -- Extensions should document their field schema, defaults, validation rules, and whether changes +- The workflow front matter is extensible. Extensions MAY define additional top-level keys without + changing the core schema above. +- Extensions SHOULD document their field schema, defaults, validation rules, and whether changes apply dynamically or require restart. -- Common extension: `server.port` (integer) enables the optional HTTP server described in Section - 13.7. #### 5.3.1 `tracker` (object) Fields: - `kind` (string) - - Required for dispatch. + - REQUIRED for dispatch. - Current supported value: `linear` - `endpoint` (string) - Default for `tracker.kind == "linear"`: `https://api.linear.app/graphql` - `api_key` (string) - - May be a literal token or `$VAR_NAME`. + - MAY be a literal token or `$VAR_NAME`. - Canonical environment variable for `tracker.kind == "linear"`: `LINEAR_API_KEY`. - If `$VAR_NAME` resolves to an empty string, treat the key as missing. - `project_slug` (string) - - Required for dispatch when `tracker.kind == "linear"`. + - REQUIRED for dispatch when `tracker.kind == "linear"`. - `active_states` (list of strings) - Default: `Todo`, `In Progress` - `terminal_states` (list of strings) @@ -360,9 +367,9 @@ Fields: Fields: -- `interval_ms` (integer or string integer) +- `interval_ms` (integer) - Default: `30000` - - Changes should be re-applied at runtime and affect future tick scheduling without restart. + - Changes SHOULD be re-applied at runtime and affect future tick scheduling without restart. #### 5.3.3 `workspace` (object) @@ -370,44 +377,48 @@ Fields: - `root` (path string or `$VAR`) - Default: `/symphony_workspaces` - - `~` and strings containing path separators are expanded. - - Bare strings without path separators are preserved as-is (relative roots are allowed but - discouraged). + - `~` is expanded. + - Relative paths are resolved relative to the directory containing `WORKFLOW.md`. + - The effective workspace root is normalized to an absolute path before use. #### 5.3.4 `hooks` (object) Fields: -- `after_create` (multiline shell script string, optional) +- `after_create` (multiline shell script string, OPTIONAL) - Runs only when a workspace directory is newly created. - Failure aborts workspace creation. -- `before_run` (multiline shell script string, optional) +- `before_run` (multiline shell script string, OPTIONAL) - Runs before each agent attempt after workspace preparation and before launching the coding agent. - Failure aborts the current attempt. -- `after_run` (multiline shell script string, optional) +- `after_run` (multiline shell script string, OPTIONAL) - Runs after each agent attempt (success, failure, timeout, or cancellation) once the workspace exists. - Failure is logged but ignored. -- `before_remove` (multiline shell script string, optional) +- `before_remove` (multiline shell script string, OPTIONAL) - Runs before workspace deletion if the directory exists. - Failure is logged but ignored; cleanup still proceeds. -- `timeout_ms` (integer, optional) +- `timeout_ms` (integer, OPTIONAL) - Default: `60000` - Applies to all workspace hooks. - - Non-positive values should be treated as invalid and fall back to the default. - - Changes should be re-applied at runtime for future hook executions. + - Invalid values fail configuration validation. + - Changes SHOULD be re-applied at runtime for future hook executions. #### 5.3.5 `agent` (object) Fields: -- `max_concurrent_agents` (integer or string integer) +- `max_concurrent_agents` (integer) - Default: `10` - - Changes should be re-applied at runtime and affect subsequent dispatch decisions. -- `max_retry_backoff_ms` (integer or string integer) + - Changes SHOULD be re-applied at runtime and affect subsequent dispatch decisions. +- `max_turns` (positive integer) + - Default: `20` + - Limits the number of coding-agent turns within one worker session. + - Invalid values fail configuration validation. +- `max_retry_backoff_ms` (integer) - Default: `300000` (5 minutes) - - Changes should be re-applied at runtime and affect future retry scheduling. + - Changes SHOULD be re-applied at runtime and affect future retry scheduling. - `max_concurrent_agents_by_state` (map `state_name -> positive integer`) - Default: empty map. - State keys are normalized (`lowercase`) for lookup. @@ -419,16 +430,16 @@ Fields: For Codex-owned config values such as `approval_policy`, `thread_sandbox`, and `turn_sandbox_policy`, supported values are defined by the targeted Codex app-server version. -Implementors should treat them as pass-through Codex config values rather than relying on a +Implementors SHOULD treat them as pass-through Codex config values rather than relying on a hand-maintained enum in this spec. To inspect the installed Codex schema, run `codex app-server generate-json-schema --out ` and inspect the relevant definitions referenced -by `v2/ThreadStartParams.json` and `v2/TurnStartParams.json`. Implementations may validate these +by `v2/ThreadStartParams.json` and `v2/TurnStartParams.json`. Implementations MAY validate these fields locally if they want stricter startup checks. - `command` (string shell command) - Default: `codex app-server` - The runtime launches this command via `bash -lc` in the workspace directory. - - The launched process must speak a compatible app-server protocol over stdio. + - The launched process MUST speak a compatible app-server protocol over stdio. - `approval_policy` (Codex `AskForApproval` value) - Default: implementation-defined. - `thread_sandbox` (Codex `SandboxMode` value) @@ -450,8 +461,8 @@ The Markdown body of `WORKFLOW.md` is the per-issue prompt template. Rendering requirements: - Use a strict template engine (Liquid-compatible semantics are sufficient). -- Unknown variables must fail rendering. -- Unknown filters must fail rendering. +- Unknown variables MUST fail rendering. +- Unknown filters MUST fail rendering. Template input variables: @@ -463,9 +474,9 @@ Template input variables: Fallback prompt behavior: -- If the workflow prompt body is empty, the runtime may use a minimal default prompt +- If the workflow prompt body is empty, the runtime MAY use a minimal default prompt (`You are working on an issue from Linear.`). -- Workflow file read/parse failures are configuration/validation errors and should not silently fall +- Workflow file read/parse failures are configuration/validation errors and SHOULD NOT silently fall back to a prompt. ### 5.5 Workflow Validation and Error Surface @@ -485,14 +496,18 @@ Dispatch gating behavior: ## 6. Configuration Specification -### 6.1 Source Precedence and Resolution Semantics +### 6.1 Configuration Resolution Pipeline + +Configuration is resolved in this order: -Configuration precedence: +1. Select the workflow file path (explicit runtime setting, otherwise cwd default). +2. Parse YAML front matter into a raw config map. +3. Apply built-in defaults for missing OPTIONAL fields. +4. Resolve `$VAR_NAME` indirection only for config values that explicitly contain `$VAR_NAME`. +5. Coerce and validate typed values. -1. Workflow file path selection (runtime setting -> cwd default). -2. YAML front matter values. -3. Environment indirection via `$VAR_NAME` inside selected YAML values. -4. Built-in defaults. +Environment variables do not globally override YAML values. They are used only when a config value +explicitly references them. Value coercion semantics: @@ -501,25 +516,27 @@ Value coercion semantics: - `$VAR` expansion for env-backed path values - Apply expansion only to values intended to be local filesystem paths; do not rewrite URIs or arbitrary shell command strings. +- Relative `workspace.root` values resolve relative to the directory containing the selected + `WORKFLOW.md`. ### 6.2 Dynamic Reload Semantics -Dynamic reload is required: +Dynamic reload is REQUIRED: -- The software should watch `WORKFLOW.md` for changes. -- On change, it should re-read and re-apply workflow config and prompt template without restart. -- The software should attempt to adjust live behavior to the new config (for example polling +- The software MUST detect `WORKFLOW.md` changes. +- On change, it MUST re-read and re-apply workflow config and prompt template without restart. +- The software MUST attempt to adjust live behavior to the new config (for example polling cadence, concurrency limits, active/terminal states, codex settings, workspace paths/hooks, and prompt content for future runs). - Reloaded config applies to future dispatch, retry scheduling, reconciliation decisions, hook execution, and agent launches. -- Implementations are not required to restart in-flight agent sessions automatically when config +- Implementations are not REQUIRED to restart in-flight agent sessions automatically when config changes. -- Extensions that manage their own listeners/resources (for example an HTTP server port change) may +- Extensions that manage their own listeners/resources (for example an HTTP server port change) MAY require restart unless the implementation explicitly supports live rebind. -- Implementations should also re-validate/reload defensively during runtime operations (for example +- Implementations SHOULD also re-validate/reload defensively during runtime operations (for example before dispatch) in case filesystem watch events are missed. -- Invalid reloads should not crash the service; keep operating with the last known good effective +- Invalid reloads MUST NOT crash the service; keep operating with the last known good effective configuration and emit an operator-visible error. ### 6.3 Dispatch Preflight Validation @@ -544,25 +561,23 @@ Validation checks: - Workflow file can be loaded and parsed. - `tracker.kind` is present and supported. - `tracker.api_key` is present after `$` resolution. -- `tracker.project_slug` is present when required by the selected tracker kind. +- `tracker.project_slug` is present when REQUIRED by the selected tracker kind. - `codex.command` is present and non-empty. -### 6.4 Config Fields Summary (Cheat Sheet) +### 6.4 Core Config Fields Summary (Cheat Sheet) This section is intentionally redundant so a coding agent can implement the config layer quickly. +Extension fields are documented in the extension section that defines them. Core conformance does +not require recognizing or validating extension fields unless that extension is implemented. -- `tracker.kind`: string, required, currently `linear` +- `tracker.kind`: string, REQUIRED, currently `linear` - `tracker.endpoint`: string, default `https://api.linear.app/graphql` when `tracker.kind=linear` - `tracker.api_key`: string or `$VAR`, canonical env `LINEAR_API_KEY` when `tracker.kind=linear` -- `tracker.project_slug`: string, required when `tracker.kind=linear` +- `tracker.project_slug`: string, REQUIRED when `tracker.kind=linear` - `tracker.active_states`: list of strings, default `["Todo", "In Progress"]` - `tracker.terminal_states`: list of strings, default `["Closed", "Cancelled", "Canceled", "Duplicate", "Done"]` - `polling.interval_ms`: integer, default `30000` -- `workspace.root`: path, default `/symphony_workspaces` -- `worker.ssh_hosts` (extension): list of SSH host strings, optional; when omitted, work runs - locally -- `worker.max_concurrent_agents_per_host` (extension): positive integer, optional; shared per-host - cap applied across configured SSH hosts +- `workspace.root`: path resolved to absolute, default `/symphony_workspaces` - `hooks.after_create`: shell script or null - `hooks.before_run`: shell script or null - `hooks.after_run`: shell script or null @@ -579,8 +594,6 @@ This section is intentionally redundant so a coding agent can implement the conf - `codex.turn_timeout_ms`: integer, default `3600000` - `codex.read_timeout_ms`: integer, default `5000` - `codex.stall_timeout_ms`: integer, default `300000` -- `server.port` (extension): integer, optional; enables the optional HTTP server, `0` may be used - for ephemeral local bind, and CLI `--port` overrides it ## 7. Orchestration State Machine @@ -612,12 +625,12 @@ claim state. Important nuance: - A successful worker exit does not mean the issue is done forever. -- The worker may continue through multiple back-to-back coding-agent turns before it exits. +- The worker MAY continue through multiple back-to-back coding-agent turns before it exits. - After each normal turn completion, the worker re-checks the tracker issue state. -- If the issue is still in an active state, the worker should start another turn on the same live +- If the issue is still in an active state, the worker SHOULD start another turn on the same live coding-agent thread in the same workspace, up to `agent.max_turns`. -- The first turn should use the full rendered task prompt. -- Continuation turns should send only continuation guidance to the existing thread, not resend the +- The first turn SHOULD use the full rendered task prompt. +- Continuation turns SHOULD send only continuation guidance to the existing thread, not resend the original task prompt that is already present in thread history. - Once the worker exits normally, the orchestrator still schedules a short continuation retry (about 1 second) so it can re-check whether the issue remains active and needs another worker @@ -675,9 +688,9 @@ Distinct terminal reasons are important because retry logic and logs differ. ### 7.4 Idempotency and Recovery Rules - The orchestrator serializes state mutations through one authority to avoid duplicate dispatch. -- `claimed` and `running` checks are required before launching any worker. +- `claimed` and `running` checks are REQUIRED before launching any worker. - Reconciliation runs before dispatch on every tick. -- Restart recovery is tracker-driven and filesystem-driven (no durable orchestrator DB required). +- Restart recovery is tracker-driven and filesystem-driven (without a durable orchestrator DB). - Startup terminal cleanup removes stale workspaces for issues already in terminal states. ## 8. Polling, Scheduling, and Reconciliation @@ -687,7 +700,7 @@ Distinct terminal reasons are important because retry logic and logs differ. At startup, the service validates config, performs startup cleanup, schedules an immediate tick, and then repeats every `polling.interval_ms`. -The effective poll interval should be updated when workflow config changes are re-applied. +The effective poll interval SHOULD be updated when workflow config changes are re-applied. Tick sequence: @@ -733,12 +746,6 @@ Per-state limit: The runtime counts issues by their current tracked state in the `running` map. -Optional SSH host limit: - -- When `worker.max_concurrent_agents_per_host` is set, each configured SSH host may run at most - that many concurrent agents at once. -- Hosts at that cap are skipped for new dispatch until capacity frees up. - ### 8.4 Retry and Backoff Retry entry creation: @@ -806,8 +813,7 @@ This prevents stale terminal workspaces from accumulating after restarts. Workspace root: -- `workspace.root` (normalized path; the current config layer expands path-like values and preserves - bare relative names) +- `workspace.root` (normalized absolute path) Per-issue workspace path: @@ -837,19 +843,19 @@ Notes: - Workspace preparation beyond directory creation (for example dependency bootstrap, checkout/sync, code generation) is implementation-defined and is typically handled via hooks. -### 9.3 Optional Workspace Population (Implementation-Defined) +### 9.3 OPTIONAL Workspace Population (Implementation-Defined) The spec does not require any built-in VCS or repository bootstrap behavior. -Implementations may populate or synchronize the workspace using implementation-defined logic and/or +Implementations MAY populate or synchronize the workspace using implementation-defined logic and/or hooks (for example `after_create` and/or `before_run`). Failure handling: - Workspace population/synchronization failures return an error for the current attempt. -- If failure happens while creating a brand-new workspace, implementations may remove the partially +- If failure happens while creating a brand-new workspace, implementations MAY remove the partially prepared directory. -- Reused workspaces should not be destructively reset on population failure unless that policy is +- Reused workspaces SHOULD NOT be destructively reset on population failure unless that policy is explicitly chosen and documented. ### 9.4 Workspace Hooks @@ -886,7 +892,7 @@ Invariant 1: Run the coding agent only in the per-issue workspace path. - Before launching the coding-agent subprocess, validate: - `cwd == workspace_path` -Invariant 2: Workspace path must stay inside workspace root. +Invariant 2: Workspace path MUST stay inside workspace root. - Normalize both paths to absolute. - Require `workspace_path` to have `workspace_root` as a prefix directory. @@ -899,17 +905,19 @@ Invariant 3: Workspace key is sanitized. ## 10. Agent Runner Protocol (Coding Agent Integration) -This section defines the language-neutral contract for integrating a coding agent app-server. +This section defines Symphony's language-neutral responsibilities when integrating a Codex +app-server. The Codex app-server protocol for the targeted Codex version is the source of truth for +protocol schemas, message payloads, transport framing, and method names. -Compatibility profile: +Protocol source of truth: -- The normative contract is message ordering, required behaviors, and the logical fields that must - be extracted (for example session IDs, completion state, approval handling, and usage/rate-limit - telemetry). -- Exact JSON field names may vary slightly across compatible app-server versions. -- Implementations should tolerate equivalent payload shapes when they carry the same logical - meaning, especially for nested IDs, approval requests, user-input-required signals, and - token/rate-limit metadata. +- Implementations MUST send messages that are valid for the targeted Codex app-server version. +- Implementations MUST consult the targeted Codex app-server documentation or generated schema + instead of treating this specification as a protocol schema. +- If this specification appears to conflict with the targeted Codex app-server protocol, the Codex + protocol controls protocol shape and transport behavior. +- Symphony-specific requirements in this section still control orchestration behavior, workspace + selection, prompt construction, continuation handling, and observability extraction. ### 10.1 Launch Contract @@ -918,107 +926,84 @@ Subprocess launch parameters: - Command: `codex.command` - Invocation: `bash -lc ` - Working directory: workspace path -- Stdout/stderr: separate streams -- Framing: line-delimited protocol messages on stdout (JSON-RPC-like JSON per line) +- Transport/framing: the protocol transport required by the targeted Codex app-server version Notes: - The default command is `codex app-server`. -- Approval policy, cwd, and prompt are expressed in the protocol messages in Section 10.2. +- Approval policy, sandbox policy, cwd, prompt input, and OPTIONAL tool declarations are supplied + using fields supported by the targeted Codex app-server version. -Recommended additional process settings: +RECOMMENDED additional process settings: - Max line size: 10 MB (for safe buffering) -### 10.2 Session Startup Handshake +### 10.2 Session Startup Responsibilities Reference: https://developers.openai.com/codex/app-server/ -The client must send these protocol messages in order: - -Illustrative startup transcript (equivalent payload shapes are acceptable if they preserve the same -semantics): - -```json -{"id":1,"method":"initialize","params":{"clientInfo":{"name":"symphony","version":"1.0"},"capabilities":{}}} -{"method":"initialized","params":{}} -{"id":2,"method":"thread/start","params":{"approvalPolicy":"","sandbox":"","cwd":"/abs/workspace"}} -{"id":3,"method":"turn/start","params":{"threadId":"","input":[{"type":"text","text":""}],"cwd":"/abs/workspace","title":"ABC-123: Example","approvalPolicy":"","sandboxPolicy":{"type":""}}} -``` - -1. `initialize` request - - Params include: - - `clientInfo` object (for example `{name, version}`) - - `capabilities` object (may be empty) - - If the targeted Codex app-server requires capability negotiation for dynamic tools, include the - necessary capability flag(s) here. - - Wait for response (`read_timeout_ms`) -2. `initialized` notification -3. `thread/start` request - - Params include: - - `approvalPolicy` = implementation-defined session approval policy value - - `sandbox` = implementation-defined session sandbox value - - `cwd` = absolute workspace path - - If optional client-side tools are implemented, include their advertised tool specs using the - protocol mechanism supported by the targeted Codex app-server version. -4. `turn/start` request - - Params include: - - `threadId` - - `input` = single text item containing rendered prompt for the first turn, or continuation - guidance for later turns on the same thread - - `cwd` - - `title` = `: ` - - `approvalPolicy` = implementation-defined turn approval policy value - - `sandboxPolicy` = implementation-defined object-form sandbox policy payload when required by - the targeted app-server version +Startup MUST follow the targeted Codex app-server contract. Symphony additionally requires the +client to: + +- Start the app-server subprocess in the per-issue workspace. +- Initialize the app-server session using the targeted Codex app-server protocol. +- Create or resume a coding-agent thread according to the targeted protocol. +- Supply the absolute per-issue workspace path as the thread/turn working directory wherever the + targeted protocol accepts cwd. +- Start the first turn with the rendered issue prompt. +- Start later in-worker continuation turns on the same live thread with continuation guidance rather + than resending the original issue prompt. +- Supply the implementation's documented approval and sandbox policy using fields supported by the + targeted protocol. +- Include issue-identifying metadata, such as `: `, when the targeted + protocol supports turn or session titles. +- Advertise implemented client-side tools using the targeted protocol. Session identifiers: -- Read `thread_id` from `thread/start` result `result.thread.id` -- Read `turn_id` from each `turn/start` result `result.turn.id` +- Extract `thread_id` from the thread identity returned by the targeted Codex app-server protocol. +- Extract `turn_id` from each turn identity returned by the targeted Codex app-server protocol. - Emit `session_id = "-"` - Reuse the same `thread_id` for all continuation turns inside one worker run ### 10.3 Streaming Turn Processing -The client reads line-delimited messages until the turn terminates. +The client processes app-server updates according to the targeted Codex app-server protocol until +the active turn terminates. Completion conditions: -- `turn/completed` -> success -- `turn/failed` -> failure -- `turn/cancelled` -> failure +- Targeted-protocol turn completion signal -> success +- Targeted-protocol turn failure signal -> failure +- Targeted-protocol turn cancellation signal -> failure - turn timeout (`turn_timeout_ms`) -> failure - subprocess exit -> failure Continuation processing: -- If the worker decides to continue after a successful turn, it should issue another `turn/start` - on the same live `threadId`. -- The app-server subprocess should remain alive across those continuation turns and be stopped only +- If the worker decides to continue after a successful turn, it SHOULD start another turn on the same + live thread using the targeted protocol. +- The app-server subprocess SHOULD remain alive across those continuation turns and be stopped only when the worker run is ending. -Line handling requirements: +Transport handling requirements: -- Read protocol messages from stdout only. -- Buffer partial stdout lines until newline arrives. -- Attempt JSON parse on complete stdout lines. -- Stderr is not part of the protocol stream: - - ignore it or log it as diagnostics - - do not attempt protocol JSON parsing on stderr +- Follow the transport and framing rules of the targeted Codex app-server version. +- For stdio-based transports, keep protocol stream handling separate from diagnostic stderr + handling unless the targeted protocol specifies otherwise. ### 10.4 Emitted Runtime Events (Upstream to Orchestrator) -The app-server client emits structured events to the orchestrator callback. Each event should +The app-server client emits structured events to the orchestrator callback. Each event SHOULD include: - `event` (enum/string) - `timestamp` (UTC timestamp) - `codex_app_server_pid` (if available) -- optional `usage` map (token counts) +- OPTIONAL `usage` map (token counts) - payload fields as needed -Important emitted events may include: +Important emitted events include, for example: - `session_started` - `startup_failed` @@ -1039,10 +1024,10 @@ Approval, sandbox, and user-input behavior is implementation-defined. Policy requirements: -- Each implementation should document its chosen approval, sandbox, and operator-confirmation +- Each implementation MUST document its chosen approval, sandbox, and operator-confirmation posture. -- Approval requests and user-input-required events must not leave a run stalled indefinitely. An - implementation should either satisfy them, surface them to an operator, auto-resolve them, or +- Approval requests and user-input-required events MUST NOT leave a run stalled indefinitely. An + implementation MAY either satisfy them, surface them to an operator, auto-resolve them, or fail the run according to its documented policy. Example high-trust behavior: @@ -1053,19 +1038,20 @@ Example high-trust behavior: Unsupported dynamic tool calls: -- Supported dynamic tool calls that are explicitly implemented and advertised by the runtime should +- Supported dynamic tool calls that are explicitly implemented and advertised by the runtime SHOULD be handled according to their extension contract. -- If the agent requests a dynamic tool call (`item/tool/call`) that is not supported, return a tool - failure response and continue the session. +- If the agent requests a dynamic tool call that is not supported, return a tool failure response + using the targeted protocol and continue the session. - This prevents the session from stalling on unsupported tool execution paths. Optional client-side tool extension: -- An implementation may expose a limited set of client-side tools to the app-server session. -- Current optional standardized tool: `linear_graphql`. -- If implemented, supported tools should be advertised to the app-server session during startup +- An implementation MAY expose a limited set of client-side tools to the app-server session. +- Current standardized optional tool: `linear_graphql`. +- If implemented, supported tools SHOULD be advertised to the app-server session during startup using the protocol mechanism supported by the targeted Codex app-server version. -- Unsupported tool names should still return a failure result and continue the session. +- Unsupported tool names SHOULD still return a failure result using the targeted protocol and + continue the session. `linear_graphql` extension contract: @@ -1083,10 +1069,10 @@ Optional client-side tool extension: } ``` -- `query` must be a non-empty string. -- `query` must contain exactly one GraphQL operation. -- `variables` is optional and, when present, must be a JSON object. -- Implementations may additionally accept a raw GraphQL query string as shorthand input. +- `query` MUST be a non-empty string. +- `query` MUST contain exactly one GraphQL operation. +- `variables` is OPTIONAL and, when present, MUST be a JSON object. +- Implementations MAY additionally accept a raw GraphQL query string as shorthand input. - Execute one GraphQL operation per tool call. - If the provided document contains multiple operations, reject the tool call as invalid input. - `operationName` selection is intentionally out of scope for this extension. @@ -1100,19 +1086,13 @@ Optional client-side tool extension: - Return the GraphQL response or error payload as structured tool output that the model can inspect in-session. -Illustrative responses (equivalent payload shapes are acceptable if they preserve the same outcome): - -```json -{"id":"","result":{"approved":true}} -{"id":"","result":{"success":false,"error":"unsupported_tool_call"}} -``` - -Hard failure on user input requirement: +User-input-required policy: -- If the agent requests user input, fail the run attempt immediately. -- The client detects this via: - - explicit method (`item/tool/requestUserInput`), or - - turn methods/flags indicating input is required. +- Implementations MUST document how targeted-protocol user-input-required signals are handled. +- A run MUST NOT stall indefinitely waiting for user input. +- A conforming implementation MAY fail the run, surface the request to an operator, satisfy it + through an approved operator channel, or auto-resolve it according to its documented policy. +- The example high-trust behavior above fails user-input-required turns immediately. ### 10.6 Timeouts and Error Mapping @@ -1122,7 +1102,7 @@ Timeouts: - `codex.turn_timeout_ms`: total turn stream timeout - `codex.stall_timeout_ms`: enforced by orchestrator based on event inactivity -Error mapping (recommended normalized categories): +Error mapping (RECOMMENDED normalized categories): - `codex_not_found` - `invalid_workspace_cwd` @@ -1152,9 +1132,9 @@ Note: ## 11. Issue Tracker Integration Contract (Linear-Compatible) -### 11.1 Required Operations +### 11.1 REQUIRED Operations -An implementation must support these tracker adapter operations: +An implementation MUST support these tracker adapter operations: 1. `fetch_candidate_issues()` - Return issues in configured active states for a configured project. @@ -1175,21 +1155,21 @@ Linear-specific requirements for `tracker.kind == "linear"`: - `tracker.project_slug` maps to Linear project `slugId` - Candidate issue query filters project using `project: { slugId: { eq: $projectSlug } }` - Issue-state refresh query uses GraphQL issue IDs with variable type `[ID!]` -- Pagination required for candidate issues +- Pagination REQUIRED for candidate issues - Page size default: `50` - Network timeout: `30000 ms` Important: - Linear GraphQL schema details can drift. Keep query construction isolated and test the exact query - fields/types required by this specification. + fields/types REQUIRED by this specification. -A non-Linear implementation may change transport details, but the normalized outputs must match the +A non-Linear implementation MAY change transport details, but the normalized outputs MUST match the domain model in Section 4. ### 11.3 Normalization Rules -Candidate issue normalization should produce fields listed in Section 4.1.1. +Candidate issue normalization SHOULD produce fields listed in Section 4.1.1. Additional normalization details: @@ -1200,7 +1180,7 @@ Additional normalization details: ### 11.4 Error Handling Contract -Recommended error categories: +RECOMMENDED error categories: - `unsupported_tracker_kind` - `missing_tracker_api_key` @@ -1226,8 +1206,8 @@ Symphony does not require first-class tracker write APIs in the orchestrator. - The service remains a scheduler/runner and tracker reader. - Workflow-specific success often means "reached the next handoff state" (for example `Human Review`) rather than tracker terminal state `Done`. -- If the optional `linear_graphql` client-side tool extension is implemented, it is still part of - the agent toolchain rather than orchestrator business logic. +- If the `linear_graphql` client-side tool extension is implemented, it is still part of the agent + toolchain rather than orchestrator business logic. ## 12. Prompt Construction and Context Assembly @@ -1237,7 +1217,7 @@ Inputs to prompt rendering: - `workflow.prompt_template` - normalized `issue` object -- optional `attempt` integer (retry/continuation metadata) +- OPTIONAL `attempt` integer (retry/continuation metadata) ### 12.2 Rendering Rules @@ -1248,7 +1228,7 @@ Inputs to prompt rendering: ### 12.3 Retry/Continuation Semantics -`attempt` should be passed to the template because the workflow prompt may provide different +`attempt` SHOULD be passed to the template because the workflow prompt can provide different instructions for: - first run (`attempt` null or absent) @@ -1266,12 +1246,12 @@ If prompt rendering fails: ### 13.1 Logging Conventions -Required context fields for issue-related logs: +REQUIRED context fields for issue-related logs: - `issue_id` - `issue_identifier` -Required context for coding-agent session lifecycle logs: +REQUIRED context for coding-agent session lifecycle logs: - `session_id` @@ -1284,22 +1264,22 @@ Message formatting requirements: ### 13.2 Logging Outputs and Sinks -The spec does not prescribe where logs must go (stderr, file, remote sink, etc.). +The spec does not prescribe where logs are written (stderr, file, remote sink, etc.). Requirements: -- Operators must be able to see startup/validation/dispatch failures without attaching a debugger. -- Implementations may write to one or more sinks. -- If a configured log sink fails, the service should continue running when possible and emit an +- Operators MUST be able to see startup/validation/dispatch failures without attaching a debugger. +- Implementations MAY write to one or more sinks. +- If a configured log sink fails, the service SHOULD continue running when possible and emit an operator-visible warning through any remaining sink. -### 13.3 Runtime Snapshot / Monitoring Interface (Optional but Recommended) +### 13.3 Runtime Snapshot / Monitoring Interface (OPTIONAL but RECOMMENDED) If the implementation exposes a synchronous runtime snapshot (for dashboards or monitoring), it -should return: +SHOULD return: - `running` (list of running session rows) -- each running row should include `turn_count` +- each running row SHOULD include `turn_count` - `retrying` (list of retry queue rows) - `codex_totals` - `input_tokens` @@ -1308,24 +1288,24 @@ should return: - `seconds_running` (aggregate runtime seconds as of snapshot time, including active sessions) - `rate_limits` (latest coding-agent rate limit payload, if available) -Recommended snapshot error modes: +RECOMMENDED snapshot error modes: - `timeout` - `unavailable` -### 13.4 Optional Human-Readable Status Surface +### 13.4 OPTIONAL Human-Readable Status Surface -A human-readable status surface (terminal output, dashboard, etc.) is optional and +A human-readable status surface (terminal output, dashboard, etc.) is OPTIONAL and implementation-defined. -If present, it should draw from orchestrator state/metrics only and must not be required for +If present, it SHOULD draw from orchestrator state/metrics only and MUST NOT be REQUIRED for correctness. ### 13.5 Session Metrics and Token Accounting Token accounting rules: -- Agent events may include token counts in multiple payload shapes. +- Agent events can include token counts in multiple payload shapes. - Prefer absolute thread totals when available, such as: - `thread/tokenUsage/updated` payloads - `total_token_usage` within token-count wrapper events @@ -1339,49 +1319,53 @@ Token accounting rules: Runtime accounting: -- Runtime should be reported as a live aggregate at snapshot/render time. -- Implementations may maintain a cumulative counter for ended sessions and add active-session +- Runtime SHOULD be reported as a live aggregate at snapshot/render time. +- Implementations MAY maintain a cumulative counter for ended sessions and add active-session elapsed time derived from `running` entries (for example `started_at`) when producing a snapshot/status view. - Add run duration seconds to the cumulative ended-session runtime when a session ends (normal exit or cancellation/termination). -- Continuous background ticking of runtime totals is not required. +- Continuous background ticking of runtime totals is not REQUIRED. Rate-limit tracking: - Track the latest rate-limit payload seen in any agent update. - Any human-readable presentation of rate-limit data is implementation-defined. -### 13.6 Humanized Agent Event Summaries (Optional) +### 13.6 Humanized Agent Event Summaries (OPTIONAL) -Humanized summaries of raw agent protocol events are optional. +Humanized summaries of raw agent protocol events are OPTIONAL. If implemented: - Treat them as observability-only output. - Do not make orchestrator logic depend on humanized strings. -### 13.7 Optional HTTP Server Extension +### 13.7 OPTIONAL HTTP Server Extension -This section defines an optional HTTP interface for observability and operational control. +This section defines an OPTIONAL HTTP interface for observability and operational control. If implemented: -- The HTTP server is an extension and is not required for conformance. -- The implementation may serve server-rendered HTML or a client-side application for the dashboard. -- The dashboard/API must be observability/control surfaces only and must not become required for +- The HTTP server is an extension and is not REQUIRED for conformance. +- The implementation MAY serve server-rendered HTML or a client-side application for the dashboard. +- The dashboard/API MUST be observability/control surfaces only and MUST NOT become REQUIRED for orchestrator correctness. +Extension config: + +- `server.port` (integer, OPTIONAL) + - Enables the HTTP server extension. + - `0` requests an ephemeral port for local development and tests. + - CLI `--port` overrides `server.port` when both are present. + Enablement (extension): - Start the HTTP server when a CLI `--port` argument is provided. - Start the HTTP server when `server.port` is present in `WORKFLOW.md` front matter. -- `server.port` is extension configuration and is intentionally not part of the core front-matter - schema in Section 5.3. -- Precedence: CLI `--port` overrides `server.port` when both are present. -- `server.port` must be an integer. Positive values bind that port. `0` may be used to request an - ephemeral port for local development and tests. -- Implementations should bind loopback by default (`127.0.0.1` or host equivalent) unless explicitly +- The `server` top-level key is owned by this extension. +- Positive `server.port` values bind that port. +- Implementations SHOULD bind loopback by default (`127.0.0.1` or host equivalent) unless explicitly configured otherwise. - Changes to HTTP listener settings (for example `server.port`) do not need to hot-rebind; restart-required behavior is conformant. @@ -1389,7 +1373,7 @@ Enablement (extension): #### 13.7.1 Human-Readable Dashboard (`/`) - Host a human-readable dashboard at `/`. -- The returned document should depict the current state of the system (for example active sessions, +- The returned document SHOULD depict the current state of the system (for example active sessions, retry delays, token consumption, runtime totals, recent events, and health/error indicators). - It is up to the implementation whether this is server-generated HTML or a client-side app that consumes the JSON API below. @@ -1507,7 +1491,7 @@ Minimum endpoints: - `POST /api/v1/refresh` - Queues an immediate tracker poll + reconciliation cycle (best-effort trigger; implementations - may coalesce repeated requests). + MAY coalesce repeated requests). - Suggested request body: empty body or `{}`. - Suggested response (`202 Accepted`) shape: @@ -1522,12 +1506,12 @@ Minimum endpoints: API design notes: -- The JSON shapes above are the recommended baseline for interoperability and debugging ergonomics. -- Implementations may add fields, but should avoid breaking existing fields within a version. -- Endpoints should be read-only except for operational triggers like `/refresh`. -- Unsupported methods on defined routes should return `405 Method Not Allowed`. -- API errors should use a JSON envelope such as `{"error":{"code":"...","message":"..."}}`. -- If the dashboard is a client-side app, it should consume this API rather than duplicating state +- The JSON shapes above are the RECOMMENDED baseline for interoperability and debugging ergonomics. +- Implementations MAY add fields, but SHOULD avoid breaking existing fields within a version. +- Endpoints SHOULD be read-only except for operational triggers like `/refresh`. +- Unsupported methods on defined routes SHOULD return `405 Method Not Allowed`. +- API errors SHOULD use a JSON envelope such as `{"error":{"code":"...","message":"..."}}`. +- If the dashboard is a client-side app, it SHOULD consume this API rather than duplicating state logic. ## 14. Failure Model and Recovery Strategy @@ -1542,7 +1526,7 @@ API design notes: 2. `Workspace Failures` - Workspace directory creation failure - - Workspace population/synchronization failure (implementation-defined; may come from hooks) + - Workspace population/synchronization failure (implementation-defined; can come from hooks) - Invalid workspace path configuration - Hook timeout/failure @@ -1550,7 +1534,7 @@ API design notes: - Startup handshake failure - Turn failed/cancelled - Turn timeout - - User input requested (hard fail) + - User input requested and handled as failure by the implementation's documented policy - Subprocess exit - Stalled session (no activity) @@ -1589,6 +1573,9 @@ API design notes: ### 14.3 Partial State Recovery (Restart) Current design is intentionally in-memory for scheduler state. +Restart recovery means the service can resume useful operation by polling tracker state and reusing +preserved workspaces. It does not mean retry timers, running sessions, or live worker state survive +process restart. After restart: @@ -1604,7 +1591,8 @@ After restart: Operators can control behavior by: - Editing `WORKFLOW.md` (prompt and most runtime settings). -- `WORKFLOW.md` changes should be detected and re-applied automatically without restart. +- `WORKFLOW.md` changes are detected and re-applied automatically without restart according to + Section 6.2. - Changing issue states in the tracker: - terminal state -> running session is stopped and workspace cleaned when reconciled - non-active state -> running session is stopped without cleanup @@ -1619,9 +1607,9 @@ Each implementation defines its own trust boundary. Operational safety requirements: -- Implementations should state clearly whether they are intended for trusted environments, more +- Implementations SHOULD state clearly whether they are intended for trusted environments, more restrictive environments, or both. -- Implementations should state clearly whether they rely on auto-approved actions, operator +- Implementations SHOULD state clearly whether they rely on auto-approved actions, operator approvals, stricter sandboxing, or some combination of those controls. - Workspace isolation and path validation are important baseline controls, but they are not a substitute for whatever approval and sandbox policy an implementation chooses. @@ -1630,11 +1618,11 @@ Operational safety requirements: Mandatory: -- Workspace path must remain under configured workspace root. -- Coding-agent cwd must be the per-issue workspace path for the current run. -- Workspace directory names must use sanitized identifiers. +- Workspace path MUST remain under configured workspace root. +- Coding-agent cwd MUST be the per-issue workspace path for the current run. +- Workspace directory names MUST use sanitized identifiers. -Recommended additional hardening for ports: +RECOMMENDED additional hardening for ports: - Run under a dedicated OS user. - Restrict workspace root permissions. @@ -1654,20 +1642,20 @@ Implications: - Hooks are fully trusted configuration. - Hooks run inside the workspace directory. -- Hook output should be truncated in logs. -- Hook timeouts are required to avoid hanging the orchestrator. +- Hook output SHOULD be truncated in logs. +- Hook timeouts are REQUIRED to avoid hanging the orchestrator. ### 15.5 Harness Hardening Guidance -Running Codex agents against repositories, issue trackers, and other inputs that may contain +Running Codex agents against repositories, issue trackers, and other inputs that can contain sensitive data or externally-controlled content can be dangerous. A permissive deployment can lead to data leaks, destructive mutations, or full machine compromise if the agent is induced to execute harmful commands or use overly-powerful integrations. -Implementations should explicitly evaluate their own risk profile and harden the execution harness +Implementations SHOULD explicitly evaluate their own risk profile and harden the execution harness where appropriate. This specification intentionally does not mandate a single hardening posture, but -ports should not assume that tracker data, repository contents, prompt inputs, or tool arguments are -fully trustworthy just because they originate inside a normal workflow. +implementations SHOULD NOT assume that tracker data, repository contents, prompt inputs, or tool +arguments are fully trustworthy just because they originate inside a normal workflow. Possible hardening measures include: @@ -1677,12 +1665,12 @@ Possible hardening measures include: separate credentials beyond the built-in Codex policy controls. - Filtering which Linear issues, projects, teams, labels, or other tracker sources are eligible for dispatch so untrusted or out-of-scope tasks do not automatically reach the agent. -- Narrowing the optional `linear_graphql` tool so it can only read or mutate data inside the +- Narrowing the `linear_graphql` tool so it can only read or mutate data inside the intended project scope, rather than exposing general workspace-wide tracker access. - Reducing the set of client-side tools, credentials, filesystem paths, and network destinations available to the agent to the minimum needed for the workflow. -The correct controls are deployment-specific, but implementations should document them clearly and +The correct controls are deployment-specific, but implementations SHOULD document them clearly and treat harness hardening as part of the core safety model rather than an optional afterthought. ## 16. Reference Algorithms (Language-Agnostic) @@ -1926,15 +1914,15 @@ on_retry_timer(issue_id, state): ## 17. Test and Validation Matrix -A conforming implementation should include tests that cover the behaviors defined in this +A conforming implementation SHOULD include tests that cover the behaviors defined in this specification. Validation profiles: -- `Core Conformance`: deterministic tests required for all conforming implementations. -- `Extension Conformance`: required only for optional features that an implementation chooses to +- `Core Conformance`: deterministic tests REQUIRED for all conforming implementations. +- `Extension Conformance`: REQUIRED only for OPTIONAL features that an implementation chooses to ship. -- `Real Integration Profile`: environment-dependent smoke/integration checks recommended before +- `Real Integration Profile`: environment-dependent smoke/integration checks RECOMMENDED before production use. Unless otherwise noted, Sections 17.1 through 17.7 are `Core Conformance`. Bullets that begin with @@ -1951,7 +1939,7 @@ Unless otherwise noted, Sections 17.1 through 17.7 are `Core Conformance`. Bulle - Missing `WORKFLOW.md` returns typed error - Invalid YAML front matter returns typed error - Front matter non-map returns typed error -- Config defaults apply when optional values are missing +- Config defaults apply when OPTIONAL values are missing - `tracker.kind` validation enforces currently supported kind (`linear`) - `tracker.api_key` works (including `$VAR` indirection) - `$VAR` resolution works for tracker API key and path values @@ -1968,8 +1956,7 @@ Unless otherwise noted, Sections 17.1 through 17.7 are `Core Conformance`. Bulle - Existing workspace directory is reused - Existing non-directory path at workspace location is handled safely (replace or fail per implementation policy) -- Optional workspace population/synchronization errors are surfaced -- Temporary artifacts (`tmp`, `.elixir_ls`) are removed during prep +- OPTIONAL workspace population/synchronization errors are surfaced - `after_create` hook runs only on new workspace creation - `before_run` hook runs before each attempt and failure/timeouts abort the current attempt - `after_run` hook runs after each attempt and failure/timeouts are logged and ignored @@ -2011,26 +1998,26 @@ Unless otherwise noted, Sections 17.1 through 17.7 are `Core Conformance`. Bulle ### 17.5 Coding-Agent App-Server Client - Launch command uses workspace cwd and invokes `bash -lc ` -- Startup handshake sends `initialize`, `initialized`, `thread/start`, `turn/start` -- `initialize` includes client identity/capabilities payload required by the targeted Codex - app-server protocol +- Session startup follows the targeted Codex app-server protocol. +- Client identity/capability payloads are valid when the targeted Codex app-server protocol requires + them. - Policy-related startup payloads use the implementation's documented approval/sandbox settings -- `thread/start` and `turn/start` parse nested IDs and emit `session_started` +- Thread and turn identities exposed by the targeted protocol are extracted and used to emit + `session_started` - Request/response read timeout is enforced - Turn timeout is enforced -- Partial JSON lines are buffered until newline -- Stdout and stderr are handled separately; protocol JSON is parsed from stdout only -- Non-JSON stderr lines are logged but do not crash parsing +- Transport framing required by the targeted protocol is handled correctly +- For stdio-based transports, diagnostic stderr handling is kept separate from the protocol stream - Command/file-change approvals are handled according to the implementation's documented policy - Unsupported dynamic tool calls are rejected without stalling the session - User input requests are handled according to the implementation's documented policy and do not stall indefinitely -- Usage and rate-limit payloads are extracted from nested payload shapes -- Compatible payload variants for approvals, user-input-required signals, and usage/rate-limit - telemetry are accepted when they preserve the same logical meaning -- If optional client-side tools are implemented, the startup handshake advertises the supported tool - specs required for discovery by the targeted app-server version -- If the optional `linear_graphql` client-side tool extension is implemented: +- Usage and rate-limit telemetry exposed by the targeted protocol is extracted +- Approval, user-input-required, usage, and rate-limit signals are interpreted according to the + targeted protocol +- If client-side tools are implemented, session startup advertises the supported tool specs + using the targeted app-server protocol +- If the `linear_graphql` client-side tool extension is implemented: - the tool is advertised to the session - valid `query` / `variables` inputs execute against configured Linear auth - top-level GraphQL `errors` produce `success=false` while preserving the GraphQL body @@ -2050,24 +2037,24 @@ Unless otherwise noted, Sections 17.1 through 17.7 are `Core Conformance`. Bulle ### 17.7 CLI and Host Lifecycle -- CLI accepts an optional positional workflow path argument (`path-to-WORKFLOW.md`) +- CLI accepts a positional workflow path argument (`path-to-WORKFLOW.md`) - CLI uses `./WORKFLOW.md` when no workflow path argument is provided - CLI errors on nonexistent explicit workflow path or missing default `./WORKFLOW.md` - CLI surfaces startup failure cleanly - CLI exits with success when application starts and shuts down normally - CLI exits nonzero when startup fails or the host process exits abnormally -### 17.8 Real Integration Profile (Recommended) +### 17.8 Real Integration Profile (RECOMMENDED) -These checks are recommended for production readiness and may be skipped in CI when credentials, +These checks are RECOMMENDED for production readiness and MAY be skipped in CI when credentials, network access, or external service permissions are unavailable. - A real tracker smoke test can be run with valid credentials supplied by `LINEAR_API_KEY` or a documented local bootstrap mechanism (for example `~/.linear_api_key`). -- Real integration tests should use isolated test identifiers/workspaces and clean up tracker +- Real integration tests SHOULD use isolated test identifiers/workspaces and clean up tracker artifacts when practical. -- A skipped real-integration test should be reported as skipped, not silently treated as passed. -- If a real-integration profile is explicitly enabled in CI or release validation, failures should +- A skipped real-integration test SHOULD be reported as skipped, not silently treated as passed. +- If a real-integration profile is explicitly enabled in CI or release validation, failures SHOULD fail that job. ## 18. Implementation Checklist (Definition of Done) @@ -2078,7 +2065,7 @@ Use the same validation profiles as Section 17: - Section 18.2 = `Extension Conformance` - Section 18.3 = `Real Integration Profile` -### 18.1 Required for Conformance +### 18.1 REQUIRED for Conformance - Workflow path selection supports explicit runtime path and cwd default - `WORKFLOW.md` loader with YAML front matter + prompt body split @@ -2097,13 +2084,13 @@ Use the same validation profiles as Section 17: - Reconciliation that stops runs on terminal/non-active tracker states - Workspace cleanup for terminal issues (startup sweep + active transition) - Structured logs with `issue_id`, `issue_identifier`, and `session_id` -- Operator-visible observability (structured logs; optional snapshot/status surface) +- Operator-visible observability (structured logs; OPTIONAL snapshot/status surface) -### 18.2 Recommended Extensions (Not Required for Conformance) +### 18.2 RECOMMENDED Extensions (Not REQUIRED for Conformance) -- Optional HTTP server honors CLI `--port` over `server.port`, uses a safe default bind host, and +- HTTP server extension honors CLI `--port` over `server.port`, uses a safe default bind host, and exposes the baseline endpoints/error semantics in Section 13.7 if shipped. -- Optional `linear_graphql` client-side tool extension exposes raw Linear GraphQL access through the +- `linear_graphql` client-side tool extension exposes raw Linear GraphQL access through the app-server session using configured Symphony auth. - TODO: Persist retry queue and session metadata across process restarts. - TODO: Make observability settings configurable in workflow front matter without prescribing UI @@ -2112,18 +2099,25 @@ Use the same validation profiles as Section 17: of only via agent tools. - TODO: Add pluggable issue tracker adapters beyond Linear. -### 18.3 Operational Validation Before Production (Recommended) +### 18.3 Operational Validation Before Production (RECOMMENDED) - Run the `Real Integration Profile` from Section 17.8 with valid credentials and network access. - Verify hook execution and workflow path resolution on the target host OS/shell environment. -- If the optional HTTP server is shipped, verify the configured port behavior and loopback/default +- If the OPTIONAL HTTP server is shipped, verify the configured port behavior and loopback/default bind expectations on the target environment. -## Appendix A. SSH Worker Extension (Optional) +## Appendix A. SSH Worker Extension (OPTIONAL) This appendix describes a common extension profile in which Symphony keeps one central orchestrator but executes worker runs on one or more remote hosts over SSH. +Extension config: + +- `worker.ssh_hosts` (list of SSH host strings, OPTIONAL) + - When omitted, work runs locally. +- `worker.max_concurrent_agents_per_host` (positive integer, OPTIONAL) + - Shared per-host cap applied across configured SSH hosts. + ### A.1 Execution Model - The orchestrator remains the single source of truth for polling, claims, retries, and @@ -2134,23 +2128,23 @@ orchestrator but executes worker runs on one or more remote hosts over SSH. - `workspace.root` is interpreted on the remote host, not on the orchestrator host. - The coding-agent app-server is launched over SSH stdio instead of as a local subprocess, so the orchestrator still owns the session lifecycle even though commands execute remotely. -- Continuation turns inside one worker lifetime should stay on the same host and workspace. -- A remote host should satisfy the same basic contract as a local worker environment: reachable +- Continuation turns inside one worker lifetime SHOULD stay on the same host and workspace. +- A remote host SHOULD satisfy the same basic contract as a local worker environment: reachable shell, writable workspace root, coding-agent executable, and any required auth or repository prerequisites. ### A.2 Scheduling Notes -- SSH hosts may be treated as a pool for dispatch. -- Implementations may prefer the previously used host on retries when that host is still +- SSH hosts MAY be treated as a pool for dispatch. +- Implementations MAY prefer the previously used host on retries when that host is still available. -- `worker.max_concurrent_agents_per_host` is an optional shared per-host cap across configured SSH +- `worker.max_concurrent_agents_per_host` is an OPTIONAL shared per-host cap across configured SSH hosts. -- When all SSH hosts are at capacity, dispatch should wait rather than silently falling back to a +- When all SSH hosts are at capacity, dispatch SHOULD wait rather than silently falling back to a different execution mode. -- Implementations may fail over to another host when the original host is unavailable before work +- Implementations MAY fail over to another host when the original host is unavailable before work has meaningfully started. -- Once a run has already produced side effects, a transparent rerun on another host should be +- Once a run has already produced side effects, a transparent rerun on another host SHOULD be treated as a new attempt, not as invisible failover. ### A.3 Problems to Consider @@ -2165,10 +2159,10 @@ orchestrator but executes worker runs on one or more remote hosts over SSH. - Remote path resolution, shell quoting, and workspace-boundary checks matter more once execution crosses a machine boundary. - Startup and failover semantics: - - Implementations should distinguish host-connectivity/startup failures from in-workspace agent + - Implementations SHOULD distinguish host-connectivity/startup failures from in-workspace agent failures so the same ticket is not accidentally re-executed on multiple hosts. - Host health and saturation: - - A dead or overloaded host should reduce available capacity, not cause duplicate execution or an + - A dead or overloaded host SHOULD reduce available capacity, not cause duplicate execution or an accidental fallback to local work. - Cleanup and observability: - Operators need to know which host owns a run, where its workspace lives, and whether cleanup From 58cf97da06d556c019ccea20c67f4f77da124bf3 Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Mon, 27 Apr 2026 15:08:58 -0700 Subject: [PATCH 10/41] fix(elixir): configure Codex app-server model via config Summary: - Update the default Symphony Codex command to select gpt-5.5 via --config model instead of the root --model flag. - Refresh README and tests so examples and argv assertions match the app-server-compatible command shape. Rationale: - Codex app-server ignores the root --model flag, so the previous command could silently use the default model instead of the intended one. - Passing model through --config keeps the workflow aligned with Codex app-server behavior and the verified gpt-5.5 setup. Tests: - mise exec -- mix format --check-formatted - mise exec -- mix test - mise exec -- mix lint - Isolated app-server smoke test with the updated command Co-authored-by: Codex --- elixir/README.md | 2 +- elixir/WORKFLOW.md | 2 +- elixir/test/symphony_elixir/core_test.exs | 4 ++-- elixir/test/symphony_elixir/workspace_and_config_test.exs | 8 ++++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/elixir/README.md b/elixir/README.md index 603b4bb000..6cb3ea98fe 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -142,7 +142,7 @@ hooks: after_create: | git clone --depth 1 "$SOURCE_REPO_URL" . codex: - command: "$CODEX_BIN app-server --model gpt-5.3-codex" + command: "$CODEX_BIN --config 'model=\"gpt-5.5\"' app-server" ``` - If `WORKFLOW.md` is missing or has invalid YAML at startup, Symphony does not boot. diff --git a/elixir/WORKFLOW.md b/elixir/WORKFLOW.md index d102b62fea..27e82cacc4 100644 --- a/elixir/WORKFLOW.md +++ b/elixir/WORKFLOW.md @@ -29,7 +29,7 @@ agent: max_concurrent_agents: 10 max_turns: 20 codex: - command: codex --config shell_environment_policy.inherit=all --config model_reasoning_effort=xhigh --model gpt-5.3-codex app-server + command: codex --config shell_environment_policy.inherit=all --config 'model="gpt-5.5"' --config model_reasoning_effort=xhigh app-server approval_policy: never thread_sandbox: workspace-write turn_sandbox_policy: diff --git a/elixir/test/symphony_elixir/core_test.exs b/elixir/test/symphony_elixir/core_test.exs index dcbe12a271..0151935781 100644 --- a/elixir/test/symphony_elixir/core_test.exs +++ b/elixir/test/symphony_elixir/core_test.exs @@ -1666,7 +1666,7 @@ defmodule SymphonyElixir.CoreTest do write_workflow_file!(Workflow.workflow_file_path(), workspace_root: workspace_root, - codex_command: "#{codex_binary} --model gpt-5.3-codex app-server" + codex_command: "#{codex_binary} --config 'model=\"gpt-5.5\"' app-server" ) issue = %Issue{ @@ -1685,7 +1685,7 @@ defmodule SymphonyElixir.CoreTest do lines = String.split(trace, "\n", trim: true) assert argv_line = Enum.find(lines, fn line -> String.starts_with?(line, "ARGV:") end) - assert String.contains?(argv_line, "--model gpt-5.3-codex app-server") + assert String.contains?(argv_line, "--config model=\"gpt-5.5\" app-server") refute String.contains?(argv_line, "--ask-for-approval never") refute String.contains?(argv_line, "--sandbox danger-full-access") after diff --git a/elixir/test/symphony_elixir/workspace_and_config_test.exs b/elixir/test/symphony_elixir/workspace_and_config_test.exs index 59ff0850b1..faa52899cf 100644 --- a/elixir/test/symphony_elixir/workspace_and_config_test.exs +++ b/elixir/test/symphony_elixir/workspace_and_config_test.exs @@ -772,8 +772,12 @@ defmodule SymphonyElixir.WorkspaceAndConfigTest do assert config.codex.read_timeout_ms == 5_000 assert config.codex.stall_timeout_ms == 300_000 - write_workflow_file!(Workflow.workflow_file_path(), codex_command: "codex app-server --model gpt-5.3-codex") - assert Config.settings!().codex.command == "codex app-server --model gpt-5.3-codex" + write_workflow_file!(Workflow.workflow_file_path(), + codex_command: "codex --config 'model=\"gpt-5.5\"' app-server" + ) + + assert Config.settings!().codex.command == + "codex --config 'model=\"gpt-5.5\"' app-server" explicit_root = Path.join( From bbef62364db25970cf0e732fc61011ab753d2604 Mon Sep 17 00:00:00 2001 From: Drew Hintz Date: Wed, 13 May 2026 22:03:45 -0500 Subject: [PATCH 11/41] [codex] Harden PR description lint workflow (#71) #### Context Restrict the PR description lint workflow so forked PR checks run with the least token access needed. #### TL;DR *Set read-only workflow permissions and stop checkout from storing credentials.* #### Summary - Add explicit contents read permission to the PR description lint workflow. - Set checkout persist-credentials to false for the lint job. #### Alternatives - Leave defaults unchanged, but explicit permissions make the token scope easier to review. #### Test Plan - [ ] `make -C elixir all` - [x] Parsed `.github/workflows/pr-description-lint.yml` with PyYAML. --- .github/workflows/pr-description-lint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr-description-lint.yml b/.github/workflows/pr-description-lint.yml index 2532abfea4..5b7743be8f 100644 --- a/.github/workflows/pr-description-lint.yml +++ b/.github/workflows/pr-description-lint.yml @@ -4,6 +4,9 @@ on: pull_request: types: [opened, edited, reopened, synchronize, ready_for_review] +permissions: + contents: read + jobs: validate-pr-description: runs-on: ubuntu-latest @@ -14,6 +17,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Set up mise tools uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3 From 3365695e85307ac50872418332cf3d2802cdda19 Mon Sep 17 00:00:00 2001 From: Danial Mirza Date: Wed, 20 May 2026 14:05:37 +0800 Subject: [PATCH 12/41] Surface input-blocked Symphony sessions (#66) #### Context Codex app-server sessions can request operator input or MCP elicitation. These should pause visibly instead of retrying until exhausted. #### TL;DR *Show input-blocked Symphony sessions in state, API, and dashboard.* #### Summary - Treat Codex input-required and MCP elicitation events as blocked sessions. - Keep blocked issues claimed until Linear state/routing changes. - Add blocked counts and per-issue blocked details to presenter payloads. - Render blocked sessions in the dashboard. - Add regression coverage for app-server and orchestrator blocked flows. #### Alternatives - Retrying was rejected because human-input blockers are not transient failures. - Moving Linear to Done was rejected because Done is post-merge terminal state. #### Test Plan - [ ] `make -C elixir all` - [x] `mise exec -- mix format` - [x] `mise exec -- mix test test/symphony_elixir/app_server_test.exs test/symphony_elixir/extensions_test.exs test/symphony_elixir/orchestrator_status_test.exs` --------- Co-authored-by: Codex --- elixir/README.md | 5 + .../lib/symphony_elixir/codex/app_server.ex | 2 + elixir/lib/symphony_elixir/orchestrator.ex | 356 +++++++++++++++--- .../live/dashboard_live.ex | 80 ++++ elixir/lib/symphony_elixir_web/presenter.ex | 83 ++-- .../test/symphony_elixir/app_server_test.exs | 65 ++++ .../test/symphony_elixir/extensions_test.exs | 53 ++- .../orchestrator_status_test.exs | 173 +++++++++ 8 files changed, 748 insertions(+), 69 deletions(-) diff --git a/elixir/README.md b/elixir/README.md index 6cb3ea98fe..b35bae2358 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -26,6 +26,11 @@ skills can make raw Linear GraphQL calls. If a claimed issue moves to a terminal state (`Done`, `Closed`, `Cancelled`, or `Duplicate`), Symphony stops the active agent for that issue and cleans up matching workspaces. +If Codex reports that operator input, approval, or MCP elicitation is required, Symphony keeps the +issue claimed and exposes it as blocked in the runtime state, JSON API, and dashboard. Blocked +entries are in memory only; restarting the orchestrator clears that blocked map, so any still-active +Linear issue can become a dispatch candidate again after restart. + ## How to use it 1. Make sure your codebase is set up to work well with agents: see diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index 9f36bc7a0c..4b46cbede7 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -1059,6 +1059,8 @@ defmodule SymphonyElixir.Codex.AppServer do Port.command(port, line) end + defp needs_input?("mcpServer/elicitation/request", payload) when is_map(payload), do: true + defp needs_input?(method, payload) when is_binary(method) and is_map(payload) do String.starts_with?(method, "turn/") && input_required_method?(method, payload) diff --git a/elixir/lib/symphony_elixir/orchestrator.ex b/elixir/lib/symphony_elixir/orchestrator.ex index 3cd814829b..0445776945 100644 --- a/elixir/lib/symphony_elixir/orchestrator.ex +++ b/elixir/lib/symphony_elixir/orchestrator.ex @@ -36,6 +36,7 @@ defmodule SymphonyElixir.Orchestrator do running: %{}, completed: MapSet.new(), claimed: MapSet.new(), + blocked: %{}, retry_attempts: %{}, codex_totals: nil, codex_rate_limits: nil @@ -129,32 +130,7 @@ defmodule SymphonyElixir.Orchestrator do state = record_session_completion_totals(state, running_entry) session_id = running_entry_session_id(running_entry) - state = - case reason do - :normal -> - Logger.info("Agent task completed for issue_id=#{issue_id} session_id=#{session_id}; scheduling active-state continuation check") - - state - |> complete_issue(issue_id) - |> schedule_issue_retry(issue_id, 1, %{ - identifier: running_entry.identifier, - delay_type: :continuation, - worker_host: Map.get(running_entry, :worker_host), - workspace_path: Map.get(running_entry, :workspace_path) - }) - - _ -> - Logger.warning("Agent task exited for issue_id=#{issue_id} session_id=#{session_id} reason=#{inspect(reason)}; scheduling retry") - - next_attempt = next_retry_attempt_from_running(running_entry) - - schedule_issue_retry(state, issue_id, next_attempt, %{ - identifier: running_entry.identifier, - error: "agent exited: #{inspect(reason)}", - worker_host: Map.get(running_entry, :worker_host), - workspace_path: Map.get(running_entry, :workspace_path) - }) - end + state = handle_agent_down(reason, state, issue_id, running_entry, session_id) Logger.info("Agent task finished for issue_id=#{issue_id} session_id=#{session_id} reason=#{inspect(reason)}") @@ -221,8 +197,57 @@ defmodule SymphonyElixir.Orchestrator do {:noreply, state} end + defp handle_agent_down(:normal, state, issue_id, running_entry, session_id) do + if input_required_blocker?(running_entry) do + block_input_required_agent_down(state, issue_id, running_entry, session_id, :normal) + else + Logger.info("Agent task completed for issue_id=#{issue_id} session_id=#{session_id}; scheduling active-state continuation check") + + state + |> complete_issue(issue_id) + |> schedule_issue_retry(issue_id, 1, %{ + identifier: running_entry.identifier, + delay_type: :continuation, + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path) + }) + end + end + + defp handle_agent_down(reason, state, issue_id, running_entry, session_id) do + if input_required_blocker?(running_entry) do + block_input_required_agent_down(state, issue_id, running_entry, session_id, reason) + else + retry_agent_down(state, issue_id, running_entry, session_id, reason) + end + end + + defp block_input_required_agent_down(state, issue_id, running_entry, session_id, reason) do + error = blocker_error(running_entry, "agent exited: #{inspect(reason)}") + + Logger.warning("Agent task blocked for issue_id=#{issue_id} issue_identifier=#{running_entry.identifier} session_id=#{session_id}: #{error}") + + block_issue_from_entry(state, issue_id, running_entry, error) + end + + defp retry_agent_down(state, issue_id, running_entry, session_id, reason) do + Logger.warning("Agent task exited for issue_id=#{issue_id} session_id=#{session_id} reason=#{inspect(reason)}; scheduling retry") + + next_attempt = next_retry_attempt_from_running(running_entry) + + schedule_issue_retry(state, issue_id, next_attempt, %{ + identifier: running_entry.identifier, + error: "agent exited: #{inspect(reason)}", + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path) + }) + end + defp maybe_dispatch(%State{} = state) do - state = reconcile_running_issues(state) + state = + state + |> reconcile_running_issues() + |> reconcile_blocked_issues() with :ok <- Config.validate!(), {:ok, issues} <- Tracker.fetch_candidate_issues(), @@ -297,6 +322,30 @@ defmodule SymphonyElixir.Orchestrator do end end + defp reconcile_blocked_issues(%State{} = state) do + blocked_ids = Map.keys(state.blocked) + + if blocked_ids == [] do + state + else + case Tracker.fetch_issue_states_by_ids(blocked_ids) do + {:ok, issues} -> + issues + |> reconcile_blocked_issue_states( + state, + active_state_set(), + terminal_state_set() + ) + |> reconcile_missing_blocked_issue_ids(blocked_ids, issues) + + {:error, reason} -> + Logger.debug("Failed to refresh blocked issue states: #{inspect(reason)}; keeping blocked issues") + + state + end + end + end + @doc false @spec reconcile_issue_states_for_test([Issue.t()], term()) :: term() def reconcile_issue_states_for_test(issues, %State{} = state) when is_list(issues) do @@ -368,6 +417,39 @@ defmodule SymphonyElixir.Orchestrator do defp reconcile_issue_state(_issue, state, _active_states, _terminal_states), do: state + defp reconcile_blocked_issue_states([], state, _active_states, _terminal_states), do: state + + defp reconcile_blocked_issue_states([issue | rest], state, active_states, terminal_states) do + reconcile_blocked_issue_states( + rest, + reconcile_blocked_issue_state(issue, state, active_states, terminal_states), + active_states, + terminal_states + ) + end + + defp reconcile_blocked_issue_state(%Issue{} = issue, state, active_states, terminal_states) do + cond do + terminal_issue_state?(issue.state, terminal_states) -> + Logger.info("Blocked issue moved to terminal state: #{issue_context(issue)} state=#{issue.state}; releasing block") + cleanup_issue_workspace(issue.identifier, blocked_issue_worker_host(state, issue.id)) + release_issue_claim(state, issue.id) + + !issue_routable_to_worker?(issue) -> + Logger.info("Blocked issue no longer routed to this worker: #{issue_context(issue)} assignee=#{inspect(issue.assignee_id)}; releasing block") + release_issue_claim(state, issue.id) + + active_issue_state?(issue.state, active_states) -> + refresh_blocked_issue_state(state, issue) + + true -> + Logger.info("Blocked issue moved to non-active state: #{issue_context(issue)} state=#{issue.state}; releasing block") + release_issue_claim(state, issue.id) + end + end + + defp reconcile_blocked_issue_state(_issue, state, _active_states, _terminal_states), do: state + defp reconcile_missing_running_issue_ids(%State{} = state, requested_issue_ids, issues) when is_list(requested_issue_ids) and is_list(issues) do visible_issue_ids = @@ -390,6 +472,28 @@ defmodule SymphonyElixir.Orchestrator do defp reconcile_missing_running_issue_ids(state, _requested_issue_ids, _issues), do: state + defp reconcile_missing_blocked_issue_ids(%State{} = state, requested_issue_ids, issues) + when is_list(requested_issue_ids) and is_list(issues) do + visible_issue_ids = + issues + |> Enum.flat_map(fn + %Issue{id: issue_id} when is_binary(issue_id) -> [issue_id] + _ -> [] + end) + |> MapSet.new() + + Enum.reduce(requested_issue_ids, state, fn issue_id, state_acc -> + if MapSet.member?(visible_issue_ids, issue_id) do + state_acc + else + Logger.info("Blocked issue no longer visible during state refresh: issue_id=#{issue_id}; releasing block") + release_issue_claim(state_acc, issue_id) + end + end) + end + + defp reconcile_missing_blocked_issue_ids(state, _requested_issue_ids, _issues), do: state + defp log_missing_running_issue(%State{} = state, issue_id) when is_binary(issue_id) do case Map.get(state.running, issue_id) do %{identifier: identifier} -> @@ -412,6 +516,16 @@ defmodule SymphonyElixir.Orchestrator do end end + defp refresh_blocked_issue_state(%State{} = state, %Issue{} = issue) do + case Map.get(state.blocked, issue.id) do + %{issue: _} = blocked_entry -> + %{state | blocked: Map.put(state.blocked, issue.id, %{blocked_entry | issue: issue})} + + _ -> + state + end + end + defp terminate_running_issue(%State{} = state, issue_id, cleanup_workspace) do case Map.get(state.running, issue_id) do nil -> @@ -425,18 +539,13 @@ defmodule SymphonyElixir.Orchestrator do cleanup_issue_workspace(identifier, worker_host) end - if is_pid(pid) do - terminate_task(pid) - end - - if is_reference(ref) do - Process.demonitor(ref, [:flush]) - end + stop_running_task(pid, ref) %{ state | running: Map.delete(state.running, issue_id), claimed: MapSet.delete(state.claimed, issue_id), + blocked: Map.delete(state.blocked, issue_id), retry_attempts: Map.delete(state.retry_attempts, issue_id) } @@ -459,11 +568,19 @@ defmodule SymphonyElixir.Orchestrator do now = DateTime.utc_now() Enum.reduce(state.running, state, fn {issue_id, running_entry}, state_acc -> - restart_stalled_issue(state_acc, issue_id, running_entry, now, timeout_ms) + maybe_restart_stalled_issue(state_acc, issue_id, running_entry, now, timeout_ms) end) end end + defp maybe_restart_stalled_issue(state, issue_id, running_entry, now, timeout_ms) do + if Map.has_key?(state.blocked, issue_id) do + state + else + restart_stalled_issue(state, issue_id, running_entry, now, timeout_ms) + end + end + defp restart_stalled_issue(state, issue_id, running_entry, now, timeout_ms) do elapsed_ms = stall_elapsed_ms(running_entry, now) @@ -471,16 +588,26 @@ defmodule SymphonyElixir.Orchestrator do identifier = Map.get(running_entry, :identifier, issue_id) session_id = running_entry_session_id(running_entry) - Logger.warning("Issue stalled: issue_id=#{issue_id} issue_identifier=#{identifier} session_id=#{session_id} elapsed_ms=#{elapsed_ms}; restarting with backoff") + if input_required_blocker?(running_entry) do + error = blocker_error(running_entry, "stalled for #{elapsed_ms}ms after Codex requested operator input") - next_attempt = next_retry_attempt_from_running(running_entry) + Logger.warning("Issue blocked: issue_id=#{issue_id} issue_identifier=#{identifier} session_id=#{session_id} elapsed_ms=#{elapsed_ms}; #{error}") - state - |> terminate_running_issue(issue_id, false) - |> schedule_issue_retry(issue_id, next_attempt, %{ - identifier: identifier, - error: "stalled for #{elapsed_ms}ms without codex activity" - }) + state + |> record_session_completion_totals(running_entry) + |> stop_and_block_issue(issue_id, running_entry, error) + else + Logger.warning("Issue stalled: issue_id=#{issue_id} issue_identifier=#{identifier} session_id=#{session_id} elapsed_ms=#{elapsed_ms}; restarting with backoff") + + next_attempt = next_retry_attempt_from_running(running_entry) + + state + |> terminate_running_issue(issue_id, false) + |> schedule_issue_retry(issue_id, next_attempt, %{ + identifier: identifier, + error: "stalled for #{elapsed_ms}ms without codex activity" + }) + end else state end @@ -504,6 +631,70 @@ defmodule SymphonyElixir.Orchestrator do defp last_activity_timestamp(_running_entry), do: nil + defp input_required_blocker?(running_entry) when is_map(running_entry) do + Map.get(running_entry, :last_codex_event) in [:turn_input_required, :approval_required] or + not is_nil(input_required_completion_outcome(Map.get(running_entry, :completion))) or + codex_message_method(Map.get(running_entry, :last_codex_message)) == + "mcpServer/elicitation/request" + end + + defp input_required_blocker?(_running_entry), do: false + + defp input_required_completion_outcome(completion) when is_map(completion) do + outcome = Map.get(completion, :outcome) || Map.get(completion, "outcome") + normalize_input_required_outcome(outcome) + end + + defp input_required_completion_outcome(_completion), do: nil + + defp normalize_input_required_outcome(outcome) + when outcome in [:input_required, :needs_input, :approval_required], + do: outcome + + defp normalize_input_required_outcome(outcome) when is_binary(outcome) do + case outcome do + "input_required" -> :input_required + "needs_input" -> :needs_input + "approval_required" -> :approval_required + _ -> nil + end + end + + defp normalize_input_required_outcome(_outcome), do: nil + + defp blocker_error(running_entry, fallback) when is_map(running_entry) do + codex_event_blocker_error(Map.get(running_entry, :last_codex_event)) || + completion_blocker_error(Map.get(running_entry, :completion)) || + codex_message_blocker_error(Map.get(running_entry, :last_codex_message)) || + fallback + end + + defp blocker_error(_running_entry, fallback), do: fallback + + defp codex_event_blocker_error(:turn_input_required), do: "codex turn requires operator input" + defp codex_event_blocker_error(:approval_required), do: "codex turn requires approval" + defp codex_event_blocker_error(_event), do: nil + + defp completion_blocker_error(completion) do + case input_required_completion_outcome(completion) do + outcome when outcome in [:input_required, :needs_input] -> "codex turn requires operator input" + :approval_required -> "codex turn requires approval" + nil -> nil + end + end + + defp codex_message_blocker_error(message) do + if codex_message_method(message) == "mcpServer/elicitation/request" do + "codex MCP elicitation requires operator input" + end + end + + defp codex_message_method(%{message: %{"method" => method}}) when is_binary(method), do: method + defp codex_message_method(%{message: %{method: method}}) when is_binary(method), do: method + defp codex_message_method(%{"method" => method}) when is_binary(method), do: method + defp codex_message_method(%{method: method}) when is_binary(method), do: method + defp codex_message_method(_message), do: nil + defp terminate_task(pid) when is_pid(pid) do case Task.Supervisor.terminate_child(SymphonyElixir.TaskSupervisor, pid) do :ok -> @@ -516,6 +707,47 @@ defmodule SymphonyElixir.Orchestrator do defp terminate_task(_pid), do: :ok + defp stop_running_task(pid, ref) do + if is_pid(pid) do + terminate_task(pid) + end + + if is_reference(ref) do + Process.demonitor(ref, [:flush]) + end + + :ok + end + + defp stop_and_block_issue(%State{} = state, issue_id, running_entry, error) do + stop_running_task(Map.get(running_entry, :pid), Map.get(running_entry, :ref)) + block_issue_from_entry(state, issue_id, running_entry, error) + end + + defp block_issue_from_entry(%State{} = state, issue_id, running_entry, error) do + blocked_entry = %{ + issue_id: issue_id, + identifier: Map.get(running_entry, :identifier, issue_id), + issue: Map.get(running_entry, :issue), + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path), + session_id: running_entry_session_id(running_entry), + error: error, + blocked_at: DateTime.utc_now(), + last_codex_message: Map.get(running_entry, :last_codex_message), + last_codex_event: Map.get(running_entry, :last_codex_event), + last_codex_timestamp: Map.get(running_entry, :last_codex_timestamp) + } + + %{ + state + | running: Map.delete(state.running, issue_id), + retry_attempts: Map.delete(state.retry_attempts, issue_id), + claimed: MapSet.put(state.claimed, issue_id), + blocked: Map.put(state.blocked, issue_id, blocked_entry) + } + end + defp choose_issues(issues, state) do active_states = active_state_set() terminal_states = terminal_state_set() @@ -553,7 +785,7 @@ defmodule SymphonyElixir.Orchestrator do defp should_dispatch_issue?( %Issue{} = issue, - %State{running: running, claimed: claimed} = state, + %State{running: running, claimed: claimed, blocked: blocked} = state, active_states, terminal_states ) do @@ -561,6 +793,7 @@ defmodule SymphonyElixir.Orchestrator do !todo_issue_blocked_by_non_terminal?(issue, terminal_states) and !MapSet.member?(claimed, issue.id) and !Map.has_key?(running, issue.id) and + !Map.has_key?(blocked, issue.id) and available_slots(state) > 0 and state_slots_available?(issue, running) and worker_slots_available?(state) @@ -879,6 +1112,12 @@ defmodule SymphonyElixir.Orchestrator do defp cleanup_issue_workspace(_identifier, _worker_host), do: :ok + defp blocked_issue_worker_host(%State{} = state, issue_id) do + state.blocked + |> Map.get(issue_id, %{}) + |> Map.get(:worker_host) + end + defp run_terminal_workspace_cleanup do case Tracker.fetch_issues_by_states(Config.settings!().tracker.terminal_states) do {:ok, issues} -> @@ -922,7 +1161,12 @@ defmodule SymphonyElixir.Orchestrator do end defp release_issue_claim(%State{} = state, issue_id) do - %{state | claimed: MapSet.delete(state.claimed, issue_id)} + %{ + state + | claimed: MapSet.delete(state.claimed, issue_id), + blocked: Map.delete(state.blocked, issue_id), + retry_attempts: Map.delete(state.retry_attempts, issue_id) + } end defp retry_delay(attempt, metadata) when is_integer(attempt) and attempt > 0 and is_map(metadata) do @@ -1140,10 +1384,29 @@ defmodule SymphonyElixir.Orchestrator do } end) + blocked = + state.blocked + |> Enum.map(fn {issue_id, metadata} -> + %{ + issue_id: issue_id, + identifier: Map.get(metadata, :identifier), + state: blocked_issue_state(metadata), + worker_host: Map.get(metadata, :worker_host), + workspace_path: Map.get(metadata, :workspace_path), + session_id: Map.get(metadata, :session_id), + error: Map.get(metadata, :error), + blocked_at: Map.get(metadata, :blocked_at), + last_codex_timestamp: Map.get(metadata, :last_codex_timestamp), + last_codex_message: Map.get(metadata, :last_codex_message), + last_codex_event: Map.get(metadata, :last_codex_event) + } + end) + {:reply, %{ running: running, retrying: retrying, + blocked: blocked, codex_totals: state.codex_totals, rate_limits: Map.get(state, :codex_rate_limits), polling: %{ @@ -1169,6 +1432,9 @@ defmodule SymphonyElixir.Orchestrator do }, state} end + defp blocked_issue_state(%{issue: %Issue{state: state}}), do: state + defp blocked_issue_state(_metadata), do: nil + defp integrate_codex_update(running_entry, %{event: event, timestamp: timestamp} = update) do token_delta = extract_token_delta(running_entry, update) codex_input_tokens = Map.get(running_entry, :codex_input_tokens, 0) diff --git a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex index a30631c113..43fc3ef642 100644 --- a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex +++ b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex @@ -91,6 +91,12 @@ defmodule SymphonyElixirWeb.DashboardLive do

Issues waiting for the next retry window.

+
+

Blocked

+

<%= @payload.counts.blocked %>

+

Issues paused for operator input or approval.

+
+

Total tokens

<%= format_int(@payload.codex_totals.total_tokens) %>

@@ -206,6 +212,80 @@ defmodule SymphonyElixirWeb.DashboardLive do <% end %> +
+
+
+

Blocked sessions

+

Issues paused because Codex requested operator input or approval.

+
+
+ + <%= if @payload.blocked == [] do %> +

No blocked sessions.

+ <% else %> +
+ + + + + + + + + + + + + + + + + + + + + +
IssueStateSessionBlocked atLast updateError
+
+ <%= entry.issue_identifier %> + JSON details +
+ 		+ + <%= entry.state || "Blocked" %> + + + <%= if entry.session_id do %> + + <% else %> + n/a + <% end %> + <%= entry.blocked_at || "n/a" %> +
+ <%= entry.last_message || to_string(entry.last_event || "n/a") %> + + <%= entry.last_event || "n/a" %> + <%= if entry.last_event_at do %> + · <%= entry.last_event_at %> + <% end %> + +
+ 		<%= entry.error || "n/a" %>
+
+ <% end %> +
+
diff --git a/elixir/lib/symphony_elixir_web/presenter.ex b/elixir/lib/symphony_elixir_web/presenter.ex index 1063cf7a64..3595cfeb33 100644 --- a/elixir/lib/symphony_elixir_web/presenter.ex +++ b/elixir/lib/symphony_elixir_web/presenter.ex @@ -15,10 +15,12 @@ defmodule SymphonyElixirWeb.Presenter do generated_at: generated_at, counts: %{ running: length(snapshot.running), - retrying: length(snapshot.retrying) + retrying: length(snapshot.retrying), + blocked: length(Map.get(snapshot, :blocked, [])) }, running: Enum.map(snapshot.running, &running_entry_payload/1), retrying: Enum.map(snapshot.retrying, &retry_entry_payload/1), + blocked: Enum.map(Map.get(snapshot, :blocked, []), &blocked_entry_payload/1), codex_totals: snapshot.codex_totals, rate_limits: snapshot.rate_limits } @@ -37,11 +39,12 @@ defmodule SymphonyElixirWeb.Presenter do %{} = snapshot -> running = Enum.find(snapshot.running, &(&1.identifier == issue_identifier)) retry = Enum.find(snapshot.retrying, &(&1.identifier == issue_identifier)) + blocked = Enum.find(Map.get(snapshot, :blocked, []), &(&1.identifier == issue_identifier)) - if is_nil(running) and is_nil(retry) do + if is_nil(running) and is_nil(retry) and is_nil(blocked) do {:error, :issue_not_found} else - {:ok, issue_payload_body(issue_identifier, running, retry)} + {:ok, issue_payload_body(issue_identifier, running, retry, blocked)} end _ -> @@ -60,14 +63,14 @@ defmodule SymphonyElixirWeb.Presenter do end end - defp issue_payload_body(issue_identifier, running, retry) do + defp issue_payload_body(issue_identifier, running, retry, blocked) do %{ issue_identifier: issue_identifier, - issue_id: issue_id_from_entries(running, retry), - status: issue_status(running, retry), + issue_id: issue_id_from_entries(running, retry, blocked), + status: issue_status(running, retry, blocked), workspace: %{ - path: workspace_path(issue_identifier, running, retry), - host: workspace_host(running, retry) + path: workspace_path(issue_identifier, running, retry, blocked), + host: workspace_host(running, retry, blocked) }, attempts: %{ restart_count: restart_count(retry), @@ -75,25 +78,26 @@ defmodule SymphonyElixirWeb.Presenter do }, running: running && running_issue_payload(running), retry: retry && retry_issue_payload(retry), + blocked: blocked && blocked_issue_payload(blocked), logs: %{ codex_session_logs: [] }, - recent_events: (running && recent_events_payload(running)) || [], - last_error: retry && retry.error, + recent_events: recent_events_payload(running || blocked), + last_error: (blocked && blocked.error) || (retry && retry.error), tracked: %{} } end - defp issue_id_from_entries(running, retry), - do: (running && running.issue_id) || (retry && retry.issue_id) + defp issue_id_from_entries(running, retry, blocked), + do: (running && running.issue_id) || (retry && retry.issue_id) || (blocked && blocked.issue_id) defp restart_count(retry), do: max(retry_attempt(retry) - 1, 0) defp retry_attempt(nil), do: 0 defp retry_attempt(retry), do: retry.attempt || 0 - defp issue_status(_running, nil), do: "running" - defp issue_status(nil, _retry), do: "retrying" - defp issue_status(_running, _retry), do: "running" + defp issue_status(running, _retry, _blocked) when not is_nil(running), do: "running" + defp issue_status(nil, retry, _blocked) when not is_nil(retry), do: "retrying" + defp issue_status(nil, nil, _blocked), do: "blocked" defp running_entry_payload(entry) do %{ @@ -128,6 +132,22 @@ defmodule SymphonyElixirWeb.Presenter do } end + defp blocked_entry_payload(entry) do + %{ + issue_id: entry.issue_id, + issue_identifier: entry.identifier, + state: entry.state, + error: entry.error, + worker_host: Map.get(entry, :worker_host), + workspace_path: Map.get(entry, :workspace_path), + session_id: entry.session_id, + blocked_at: iso8601(entry.blocked_at), + last_event: entry.last_codex_event, + last_message: summarize_message(entry.last_codex_message), + last_event_at: iso8601(entry.last_codex_timestamp) + } + end + defp running_issue_payload(running) do %{ worker_host: Map.get(running, :worker_host), @@ -157,22 +177,41 @@ defmodule SymphonyElixirWeb.Presenter do } end - defp workspace_path(issue_identifier, running, retry) do + defp blocked_issue_payload(blocked) do + %{ + worker_host: Map.get(blocked, :worker_host), + workspace_path: Map.get(blocked, :workspace_path), + session_id: blocked.session_id, + state: blocked.state, + error: blocked.error, + blocked_at: iso8601(blocked.blocked_at), + last_event: blocked.last_codex_event, + last_message: summarize_message(blocked.last_codex_message), + last_event_at: iso8601(blocked.last_codex_timestamp) + } + end + + defp workspace_path(issue_identifier, running, retry, blocked) do (running && Map.get(running, :workspace_path)) || (retry && Map.get(retry, :workspace_path)) || + (blocked && Map.get(blocked, :workspace_path)) || Path.join(Config.settings!().workspace.root, issue_identifier) end - defp workspace_host(running, retry) do - (running && Map.get(running, :worker_host)) || (retry && Map.get(retry, :worker_host)) + defp workspace_host(running, retry, blocked) do + (running && Map.get(running, :worker_host)) || + (retry && Map.get(retry, :worker_host)) || + (blocked && Map.get(blocked, :worker_host)) end - defp recent_events_payload(running) do + defp recent_events_payload(nil), do: [] + + defp recent_events_payload(entry) do [ %{ - at: iso8601(running.last_codex_timestamp), - event: running.last_codex_event, - message: summarize_message(running.last_codex_message) + at: iso8601(entry.last_codex_timestamp), + event: entry.last_codex_event, + message: summarize_message(entry.last_codex_message) } ] |> Enum.reject(&is_nil(&1.at)) diff --git a/elixir/test/symphony_elixir/app_server_test.exs b/elixir/test/symphony_elixir/app_server_test.exs index d03627f8b3..00cd2d283c 100644 --- a/elixir/test/symphony_elixir/app_server_test.exs +++ b/elixir/test/symphony_elixir/app_server_test.exs @@ -262,6 +262,71 @@ defmodule SymphonyElixir.AppServerTest do end end + test "app server treats MCP elicitation requests as hard input blockers" do + test_root = + Path.join( + System.tmp_dir!(), + "symphony-elixir-app-server-mcp-elicitation-#{System.unique_integer([:positive])}" + ) + + try do + workspace_root = Path.join(test_root, "workspaces") + workspace = Path.join(workspace_root, "MT-188") + codex_binary = Path.join(test_root, "fake-codex") + File.mkdir_p!(workspace) + + File.write!(codex_binary, """ + #!/bin/sh + count=0 + while IFS= read -r _line; do + count=$((count + 1)) + + case "$count" in + 1) + printf '%s\\n' '{"id":1,"result":{}}' + ;; + 2) + printf '%s\\n' '{"id":2,"result":{"thread":{"id":"thread-188"}}}' + ;; + 3) + printf '%s\\n' '{"id":3,"result":{"turn":{"id":"turn-188"}}}' + ;; + 4) + printf '%s\\n' '{"method":"mcpServer/elicitation/request","params":{"message":"Need operator input"}}' + ;; + *) + exit 0 + ;; + esac + done + """) + + File.chmod!(codex_binary, 0o755) + + write_workflow_file!(Workflow.workflow_file_path(), + workspace_root: workspace_root, + codex_command: "#{codex_binary} app-server" + ) + + issue = %Issue{ + id: "issue-mcp-elicitation", + identifier: "MT-188", + title: "MCP elicitation", + description: "Cannot satisfy MCP input", + state: "In Progress", + url: "https://example.org/issues/MT-188", + labels: ["backend"] + } + + assert {:error, {:turn_input_required, payload}} = + AppServer.run(workspace, "Needs MCP input", issue) + + assert payload["method"] == "mcpServer/elicitation/request" + after + File.rm_rf(test_root) + end + end + test "app server fails when command execution approval is required under safer defaults" do test_root = Path.join( diff --git a/elixir/test/symphony_elixir/extensions_test.exs b/elixir/test/symphony_elixir/extensions_test.exs index d6309c9662..789eb3d06f 100644 --- a/elixir/test/symphony_elixir/extensions_test.exs +++ b/elixir/test/symphony_elixir/extensions_test.exs @@ -342,7 +342,7 @@ defmodule SymphonyElixir.ExtensionsTest do assert state_payload == %{ "generated_at" => state_payload["generated_at"], - "counts" => %{"running" => 1, "retrying" => 1}, + "counts" => %{"running" => 1, "retrying" => 1, "blocked" => 1}, "running" => [ %{ "issue_id" => "issue-http", @@ -370,6 +370,21 @@ defmodule SymphonyElixir.ExtensionsTest do "workspace_path" => nil } ], + "blocked" => [ + %{ + "issue_id" => "issue-blocked", + "issue_identifier" => "MT-BLOCKED", + "state" => "In Progress", + "error" => "codex turn requires operator input", + "worker_host" => "dm-dev2", + "workspace_path" => "/workspaces/MT-BLOCKED", + "session_id" => "thread-blocked", + "blocked_at" => state_payload["blocked"] |> List.first() |> Map.fetch!("blocked_at"), + "last_event" => "turn_input_required", + "last_message" => "turn blocked: waiting for user input", + "last_event_at" => state_payload["blocked"] |> List.first() |> Map.fetch!("last_event_at") + } + ], "codex_totals" => %{ "input_tokens" => 4, "output_tokens" => 8, @@ -404,6 +419,7 @@ defmodule SymphonyElixir.ExtensionsTest do "tokens" => %{"input_tokens" => 4, "output_tokens" => 8, "total_tokens" => 12} }, "retry" => nil, + "blocked" => nil, "logs" => %{"codex_session_logs" => []}, "recent_events" => [], "last_error" => nil, @@ -415,6 +431,18 @@ defmodule SymphonyElixir.ExtensionsTest do assert %{"status" => "retrying", "retry" => %{"attempt" => 2, "error" => "boom"}} = json_response(conn, 200) + conn = get(build_conn(), "/api/v1/MT-BLOCKED") + + assert %{ + "status" => "blocked", + "last_error" => "codex turn requires operator input", + "blocked" => %{ + "session_id" => "thread-blocked", + "state" => "In Progress", + "error" => "codex turn requires operator input" + } + } = json_response(conn, 200) + conn = get(build_conn(), "/api/v1/MT-MISSING") assert json_response(conn, 404) == %{ @@ -542,7 +570,9 @@ defmodule SymphonyElixir.ExtensionsTest do assert html =~ "Operations Dashboard" assert html =~ "MT-HTTP" assert html =~ "MT-RETRY" + assert html =~ "MT-BLOCKED" assert html =~ "rendered" + assert html =~ "turn blocked: waiting for user input" assert html =~ "Runtime" assert html =~ "Live" assert html =~ "Offline" @@ -641,7 +671,7 @@ defmodule SymphonyElixir.ExtensionsTest do response = Req.get!("http://127.0.0.1:#{port}/api/v1/state") assert response.status == 200 - assert response.body["counts"] == %{"running" => 1, "retrying" => 1} + assert response.body["counts"] == %{"running" => 1, "retrying" => 1, "blocked" => 1} dashboard_css = Req.get!("http://127.0.0.1:#{port}/dashboard.css") assert dashboard_css.status == 200 @@ -711,6 +741,25 @@ defmodule SymphonyElixir.ExtensionsTest do error: "boom" } ], + blocked: [ + %{ + issue_id: "issue-blocked", + identifier: "MT-BLOCKED", + state: "In Progress", + error: "codex turn requires operator input", + worker_host: "dm-dev2", + workspace_path: "/workspaces/MT-BLOCKED", + session_id: "thread-blocked", + blocked_at: DateTime.utc_now(), + last_codex_event: :turn_input_required, + last_codex_message: %{ + event: :turn_input_required, + message: %{"method" => "turn/input_required"}, + timestamp: DateTime.utc_now() + }, + last_codex_timestamp: DateTime.utc_now() + } + ], codex_totals: %{input_tokens: 4, output_tokens: 8, total_tokens: 12, seconds_running: 42.5}, rate_limits: %{"primary" => %{"remaining" => 11}} } diff --git a/elixir/test/symphony_elixir/orchestrator_status_test.exs b/elixir/test/symphony_elixir/orchestrator_status_test.exs index 4326b80ce3..35a2977ede 100644 --- a/elixir/test/symphony_elixir/orchestrator_status_test.exs +++ b/elixir/test/symphony_elixir/orchestrator_status_test.exs @@ -961,6 +961,179 @@ defmodule SymphonyElixir.OrchestratorStatusTest do assert remaining_ms <= 10_500 end + test "orchestrator blocks stalled workers that are waiting on MCP elicitation" do + write_workflow_file!(Workflow.workflow_file_path(), + tracker_api_token: nil, + codex_stall_timeout_ms: 1_000 + ) + + issue_id = "issue-mcp-elicitation-stall" + orchestrator_name = Module.concat(__MODULE__, :McpElicitationBlockOrchestrator) + {:ok, pid} = Orchestrator.start_link(name: orchestrator_name) + + on_exit(fn -> + if Process.alive?(pid) do + Process.exit(pid, :normal) + end + end) + + worker_pid = + spawn(fn -> + receive do + :done -> :ok + end + end) + + stale_activity_at = DateTime.add(DateTime.utc_now(), -5, :second) + initial_state = :sys.get_state(pid) + + running_entry = %{ + pid: worker_pid, + ref: make_ref(), + identifier: "MT-MCP", + issue: %Issue{id: issue_id, identifier: "MT-MCP", state: "In Progress"}, + worker_host: "dm-dev2", + workspace_path: "/workspaces/MT-MCP", + session_id: "thread-mcp-turn-mcp", + last_codex_message: %{ + event: :notification, + message: %{"method" => "mcpServer/elicitation/request"}, + timestamp: stale_activity_at + }, + last_codex_timestamp: stale_activity_at, + last_codex_event: :notification, + started_at: stale_activity_at + } + + :sys.replace_state(pid, fn _ -> + initial_state + |> Map.put(:running, %{issue_id => running_entry}) + |> Map.put(:claimed, MapSet.put(initial_state.claimed, issue_id)) + end) + + send(pid, :tick) + Process.sleep(100) + state = :sys.get_state(pid) + + refute Process.alive?(worker_pid) + refute Map.has_key?(state.running, issue_id) + refute Map.has_key?(state.retry_attempts, issue_id) + assert MapSet.member?(state.claimed, issue_id) + + assert %{ + identifier: "MT-MCP", + error: "codex MCP elicitation requires operator input", + worker_host: "dm-dev2", + workspace_path: "/workspaces/MT-MCP" + } = state.blocked[issue_id] + + assert %{blocked: [%{identifier: "MT-MCP", error: "codex MCP elicitation requires operator input"}]} = + Orchestrator.snapshot(orchestrator_name, 1_000) + end + + test "orchestrator blocks failed workers after app-server reports input required" do + write_workflow_file!(Workflow.workflow_file_path(), tracker_api_token: nil) + + issue_id = "issue-input-required" + orchestrator_name = Module.concat(__MODULE__, :InputRequiredBlockOrchestrator) + {:ok, pid} = Orchestrator.start_link(name: orchestrator_name) + + on_exit(fn -> + if Process.alive?(pid) do + Process.exit(pid, :normal) + end + end) + + ref = make_ref() + started_at = DateTime.utc_now() + initial_state = :sys.get_state(pid) + + running_entry = %{ + pid: self(), + ref: ref, + identifier: "MT-INPUT", + issue: %Issue{id: issue_id, identifier: "MT-INPUT", state: "In Progress"}, + session_id: "thread-input-turn-input", + last_codex_message: %{ + event: :turn_input_required, + message: %{"method" => "mcpServer/elicitation/request"}, + timestamp: started_at + }, + last_codex_timestamp: started_at, + last_codex_event: :turn_input_required, + started_at: started_at + } + + :sys.replace_state(pid, fn _ -> + initial_state + |> Map.put(:running, %{issue_id => running_entry}) + |> Map.put(:claimed, MapSet.put(initial_state.claimed, issue_id)) + end) + + send(pid, {:DOWN, ref, :process, self(), {:shutdown, :input_required}}) + Process.sleep(50) + state = :sys.get_state(pid) + + refute Map.has_key?(state.running, issue_id) + refute Map.has_key?(state.retry_attempts, issue_id) + assert MapSet.member?(state.claimed, issue_id) + + assert %{ + identifier: "MT-INPUT", + error: "codex turn requires operator input" + } = state.blocked[issue_id] + end + + test "orchestrator blocks normal worker exits after input required completion" do + write_workflow_file!(Workflow.workflow_file_path(), tracker_api_token: nil) + + issue_id = "issue-input-required-normal" + orchestrator_name = Module.concat(__MODULE__, :InputRequiredNormalBlockOrchestrator) + {:ok, pid} = Orchestrator.start_link(name: orchestrator_name) + + on_exit(fn -> + if Process.alive?(pid) do + Process.exit(pid, :normal) + end + end) + + ref = make_ref() + initial_state = :sys.get_state(pid) + + running_entry = %{ + pid: self(), + ref: ref, + identifier: "MT-INPUT-NORMAL", + issue: %Issue{id: issue_id, identifier: "MT-INPUT-NORMAL", state: "In Progress"}, + session_id: "thread-input-normal", + completion: %{outcome: :input_required}, + last_codex_message: nil, + last_codex_timestamp: nil, + last_codex_event: nil, + started_at: DateTime.utc_now() + } + + :sys.replace_state(pid, fn _ -> + initial_state + |> Map.put(:running, %{issue_id => running_entry}) + |> Map.put(:claimed, MapSet.put(initial_state.claimed, issue_id)) + end) + + send(pid, {:DOWN, ref, :process, self(), :normal}) + Process.sleep(50) + state = :sys.get_state(pid) + + refute Map.has_key?(state.running, issue_id) + refute Map.has_key?(state.retry_attempts, issue_id) + refute MapSet.member?(state.completed, issue_id) + assert MapSet.member?(state.claimed, issue_id) + + assert %{ + identifier: "MT-INPUT-NORMAL", + error: "codex turn requires operator input" + } = state.blocked[issue_id] + end + test "status dashboard renders offline marker to terminal" do rendered = ExUnit.CaptureIO.capture_io(fn -> From 2c1851830477434100fdb8980fcc1fce1a8af81d Mon Sep 17 00:00:00 2001 From: Alex Kotliarskyi Date: Wed, 20 May 2026 15:28:54 -0700 Subject: [PATCH 13/41] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a8e006d5b..e0f24a0484 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ Symphony turns project work into isolated, autonomous implementation runs, allowing teams to manage work instead of supervising coding agents. -[![Symphony demo video preview](.github/media/symphony-demo-poster.jpg)](.github/media/symphony-demo.mp4) +[![Symphony demo video preview](.github/media/symphony-demo-poster.jpg)](https://player.vimeo.com/video/1186371009?h=5626e4b899) -_In this [demo video](.github/media/symphony-demo.mp4), Symphony monitors a Linear board for work and spawns agents to handle the tasks. The agents complete the tasks and provide proof of work: CI status, PR review feedback, complexity analysis, and walkthrough videos. When accepted, the agents land the PR safely. Engineers do not need to supervise Codex; they can manage the work at a higher level._ +_In this [demo video](https://player.vimeo.com/video/1186371009?h=5626e4b899), Symphony monitors a Linear board for work and spawns agents to handle the tasks. The agents complete the tasks and provide proof of work: CI status, PR review feedback, complexity analysis, and walkthrough videos. When accepted, the agents land the PR safely. Engineers do not need to supervise Codex; they can manage the work at a higher level._ > [!WARNING] > Symphony is a low-key engineering preview for testing in trusted environments. From f577cb5a1e971f04ebcfcdd06e72e722ab4b2ebe Mon Sep 17 00:00:00 2001 From: poon-oai Date: Fri, 29 May 2026 00:53:26 -0700 Subject: [PATCH 14/41] docs: add SD-4 Symphony smoke-test note #### Context Smoke-test ticket SD-4 asks for an isolated doc proving Symphony picked up and handled the issue. #### TL;DR *Adds the SD-4 Symphony smoke-test note.* #### Summary - Adds `docs/symphony-smoke-test-one.md`. - Keeps the change docs-only and limited to the smoke-test note. - Mentions Jira issue key `SD-4`. #### Alternatives - Leave no repo artifact; rejected because the ticket asks for a committed smoke-test doc. #### Test Plan - [x] `make -C elixir all` via GitHub Actions `make-all` - [x] `test -f docs/symphony-smoke-test-one.md && rg "SD-4|Symphony" docs/symphony-smoke-test-one.md && git diff --check` - [x] `mix pr_body.check --file ` --- docs/symphony-smoke-test-one.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/symphony-smoke-test-one.md diff --git a/docs/symphony-smoke-test-one.md b/docs/symphony-smoke-test-one.md new file mode 100644 index 0000000000..ff9f057a3f --- /dev/null +++ b/docs/symphony-smoke-test-one.md @@ -0,0 +1,3 @@ +# Symphony Smoke Test + +Symphony picked up Jira issue SD-4 and handled it by adding this isolated smoke-test doc. From c5261d12101b02e0045ca84701eed0c4be367387 Mon Sep 17 00:00:00 2001 From: poon-oai Date: Fri, 29 May 2026 13:26:53 -0700 Subject: [PATCH 15/41] docs: add board review smoke test note #### Context Adds the requested SD-6 smoke-test note so the Symphony board review workflow can validate a minimal docs-only change. #### TL;DR *Add a short board review smoke-test markdown note.* #### Summary - Added docs/symphony-smoke-board-review.md with a short heading and note. - Kept the change isolated to one new docs file. #### Alternatives - No code or app behavior changes were made because the ticket scope is docs-only. #### Test Plan - [x] `make -C elixir all` via GitHub Actions `make-all` rerun - [x] `mix pr_body.check --file <(gh pr view 79 --repo openai/symphony --json body -q .body)` - [x] `git status --short` shows only `?? docs/symphony-smoke-board-review.md` --- docs/symphony-smoke-board-review.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/symphony-smoke-board-review.md diff --git a/docs/symphony-smoke-board-review.md b/docs/symphony-smoke-board-review.md new file mode 100644 index 0000000000..0eaab456b7 --- /dev/null +++ b/docs/symphony-smoke-board-review.md @@ -0,0 +1,3 @@ +# Symphony Board Review Smoke Test + +This brief note confirms this is a Symphony smoke test for the board review workflow. From 68a18a7d4d07e8c9ebf06df95014417b303ee840 Mon Sep 17 00:00:00 2001 From: yansenzhou-oai Date: Sun, 31 May 2026 19:42:57 -0700 Subject: [PATCH 16/41] [codex] Add Codex thread links and Linear comment resumes (#84) #### Context Need Symphony to keep a first-class Codex thread loop for Linear issues, including human follow-up comments and terminal cleanup. #### TL;DR *Adds Codex thread links, thread archiving, and Linear comment resume support.* #### Summary - Add Codex app-server proxy support with thread owner metadata and archive calls. - Track thread IDs, comment cursors, and blocked human-input state in the orchestrator. - Fetch Linear comments and resume blocked or continued runs with human follow-up prompts. - Document the workflow/spec updates and cover stale empty workspace repair. #### Alternatives - Keep comments entirely inside Codex workpads, but Linear replies stay invisible to Symphony. - Leave thread handling as plain dashboard text, but `codex://` links make app handoff direct. #### Test Plan - [x] `make -C elixir all` - [x] `git diff --check` - [x] `rg -n "lin_api_|ZtX3ZP" .` --- README.md | 4 + SPEC.md | 39 +- elixir/README.md | 35 +- elixir/WORKFLOW.md | 57 +- elixir/lib/symphony_elixir/agent_runner.ex | 37 +- .../lib/symphony_elixir/codex/app_server.ex | 514 +++++++++++++++++- elixir/lib/symphony_elixir/config.ex | 12 +- elixir/lib/symphony_elixir/config/schema.ex | 8 + elixir/lib/symphony_elixir/linear/client.ex | 62 ++- elixir/lib/symphony_elixir/linear/comment.ex | 116 ++++ elixir/lib/symphony_elixir/linear/issue.ex | 2 + elixir/lib/symphony_elixir/orchestrator.ex | 285 +++++++++- elixir/lib/symphony_elixir/prompt_builder.ex | 65 ++- elixir/lib/symphony_elixir/workspace.ex | 16 +- .../live/dashboard_live.ex | 24 + elixir/lib/symphony_elixir_web/presenter.ex | 19 +- elixir/mix.exs | 2 +- elixir/test/support/test_support.exs | 13 + .../test/symphony_elixir/app_server_test.exs | 402 +++++++++++++- elixir/test/symphony_elixir/core_test.exs | 113 +++- .../test/symphony_elixir/extensions_test.exs | 17 +- .../symphony_elixir/linear_comment_test.exs | 32 ++ .../orchestrator_status_test.exs | 150 +++++ .../workspace_and_config_test.exs | 70 ++- 24 files changed, 1982 insertions(+), 112 deletions(-) create mode 100644 elixir/lib/symphony_elixir/linear/comment.ex create mode 100644 elixir/test/symphony_elixir/linear_comment_test.exs diff --git a/README.md b/README.md index e0f24a0484..23f092f826 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ work instead of supervising coding agents. _In this [demo video](https://player.vimeo.com/video/1186371009?h=5626e4b899), Symphony monitors a Linear board for work and spawns agents to handle the tasks. The agents complete the tasks and provide proof of work: CI status, PR review feedback, complexity analysis, and walkthrough videos. When accepted, the agents land the PR safely. Engineers do not need to supervise Codex; they can manage the work at a higher level._ +The reference implementation can also keep the Codex thread attached to the issue lifecycle: it +surfaces thread links, resumes from human Linear comments, and archives threads after terminal +issues are cleaned up. + > [!WARNING] > Symphony is a low-key engineering preview for testing in trusted environments. diff --git a/SPEC.md b/SPEC.md index adb8bc59a8..25ff670d5b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -36,8 +36,10 @@ stricter approvals or sandboxing. Important boundary: - Symphony is a scheduler/runner and tracker reader. -- Ticket writes (state transitions, comments, PR links) are typically performed by the coding agent - using tools available in the workflow/runtime environment. +- Most ticket writes (state transitions, progress comments, PR links) are typically performed by + the coding agent using tools available in the workflow/runtime environment. Implementations MAY + also perform narrowly scoped service comments that are part of orchestration, such as asking for + missing human input. - A successful run can end at a workflow-defined handoff state (for example `Human Review`), not necessarily `Done`. @@ -50,6 +52,8 @@ Important boundary: - Create deterministic per-issue workspaces and preserve them across runs. - Stop active runs when issue state changes make them ineligible. - Recover from transient failures with exponential backoff. +- Carry fresh tracker comments into continuation turns when an implementation supports human input + through the issue tracker. - Load runtime behavior from a repository-owned `WORKFLOW.md` contract. - Expose operator-visible observability (at minimum structured logs). - Support tracker/filesystem-driven restart recovery without requiring a persistent database; exact @@ -84,6 +88,7 @@ Important boundary: - Fetches candidate issues in active states. - Fetches current states for specific issue IDs (reconciliation). - Fetches terminal-state issues during startup cleanup. + - Fetches issue comments when the implementation uses tracker comments as human input. - Normalizes tracker payloads into a stable issue model. 4. `Orchestrator` @@ -173,6 +178,15 @@ Fields: - `id` (string or null) - `identifier` (string or null) - `state` (string or null) +- `comments` (list of comment refs, OPTIONAL) + - Present when the tracker adapter fetches comments for human input. + - Each comment ref contains: + - `id` (string) + - `body` (string) + - `author_id` (string or null) + - `author_name` (string or null) + - `created_at` (timestamp or null) + - `updated_at` (timestamp or null) - `created_at` (timestamp or null) - `updated_at` (timestamp or null) @@ -632,9 +646,14 @@ Important nuance: - The first turn SHOULD use the full rendered task prompt. - Continuation turns SHOULD send only continuation guidance to the existing thread, not resend the original task prompt that is already present in thread history. +- If new human tracker comments are available, continuation guidance SHOULD include those comments + as fresh input and keep using the existing coding-agent thread/workspace. - Once the worker exits normally, the orchestrator still schedules a short continuation retry (about 1 second) so it can re-check whether the issue remains active and needs another worker session. +- If the agent stops because it needs human input, an implementation MAY retain the workspace and + thread metadata in the retry queue, post a tracker comment asking for the input, and resume the + same thread when a newer human comment appears. ### 7.2 Run Attempt Lifecycle @@ -795,6 +814,8 @@ Part B: Tracker state refresh - If tracker state is terminal: terminate worker and clean workspace. - If tracker state is still active: update the in-memory issue snapshot. - If tracker state is neither active nor terminal: terminate worker without workspace cleanup. +- If tracker comments are used as human input, reconciliation MAY also capture newer human comments + for the running worker or for a blocked retry entry. - If state refresh fails, keep workers running and try again on the next tick. ### 8.6 Startup Terminal Workspace Cleanup @@ -965,6 +986,10 @@ Session identifiers: - Extract `turn_id` from each turn identity returned by the targeted Codex app-server protocol. - Emit `session_id = "-"` - Reuse the same `thread_id` for all continuation turns inside one worker run +- If the targeted coding-agent surface supports thread URLs, status surfaces SHOULD expose a link + derived from `thread_id`. +- If the targeted coding-agent surface supports archiving threads, implementations MAY archive the + thread when the associated issue reaches a terminal state. ### 10.3 Streaming Turn Processing @@ -1175,6 +1200,8 @@ Additional normalization details: - `labels` -> lowercase strings - `blocked_by` -> derived from inverse relations where relation type is `blocks` +- `comments` -> chronological tracker comments when fetched; implementation-defined filters MAY + exclude service-owned workpad or orchestration marker comments from human-input handling - `priority` -> integer only (non-integers become null) - `created_at` and `updated_at` -> parse ISO-8601 timestamps @@ -1199,13 +1226,15 @@ Orchestrator behavior on tracker errors: ### 11.5 Tracker Writes (Important Boundary) -Symphony does not require first-class tracker write APIs in the orchestrator. +Symphony does not require broad first-class tracker write APIs in the orchestrator. - Ticket mutations (state transitions, comments, PR metadata) are typically handled by the coding agent using tools defined by the workflow prompt. - The service remains a scheduler/runner and tracker reader. - Workflow-specific success often means "reached the next handoff state" (for example `Human Review`) rather than tracker terminal state `Done`. +- Implementations MAY use narrowly scoped tracker writes for orchestration-only notices, such as a + marked comment requesting human input before resuming the same thread. - If the `linear_graphql` client-side tool extension is implemented, it is still part of the agent toolchain rather than orchestrator business logic. @@ -2095,8 +2124,8 @@ Use the same validation profiles as Section 17: - TODO: Persist retry queue and session metadata across process restarts. - TODO: Make observability settings configurable in workflow front matter without prescribing UI implementation details. -- TODO: Add first-class tracker write APIs (comments/state transitions) in the orchestrator instead - of only via agent tools. +- TODO: Add broader first-class tracker write APIs (state transitions, PR metadata, editable + progress comments) in the orchestrator instead of only via agent tools. - TODO: Add pluggable issue tracker adapters beyond Linear. ### 18.3 Operational Validation Before Production (RECOMMENDED) diff --git a/elixir/README.md b/elixir/README.md index b35bae2358..5cade8fd49 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -16,7 +16,7 @@ This directory contains the current Elixir/OTP implementation of Symphony, based 1. Polls Linear for candidate work 2. Creates a workspace per issue 3. Launches Codex in [App Server mode](https://developers.openai.com/codex/app-server/) inside the - workspace + workspace, or connects to an existing app-server through `codex app-server proxy` 4. Sends a workflow prompt to Codex 5. Keeps Codex working on the issue until the work is done @@ -24,12 +24,19 @@ During app-server sessions, Symphony also serves a client-side `linear_graphql` skills can make raw Linear GraphQL calls. If a claimed issue moves to a terminal state (`Done`, `Closed`, `Cancelled`, or `Duplicate`), -Symphony stops the active agent for that issue and cleans up matching workspaces. +Symphony stops the active agent for that issue, archives the matching Codex thread when known, and +cleans up matching workspaces. If Codex reports that operator input, approval, or MCP elicitation is required, Symphony keeps the issue claimed and exposes it as blocked in the runtime state, JSON API, and dashboard. Blocked -entries are in memory only; restarting the orchestrator clears that blocked map, so any still-active -Linear issue can become a dispatch candidate again after restart. +entries keep the Codex `thread_id`, workspace path, and a Linear comment cursor. Symphony posts a +marked "Codex needs input" comment on the issue when possible. When a new non-workpad Linear +comment appears after the cursor, Symphony resumes the same Codex thread and worktree with that +comment as fresh human input. + +Linear comments added while a turn is actively running are queued in memory and included in the +next retry/continuation that Symphony starts. The Codex app remains the rich console for taking over +directly; Linear is the async conversation surface for issue-specific feedback. ## How to use it @@ -105,6 +112,9 @@ agent: max_turns: 20 codex: command: codex app-server + model: gpt-5 + reasoning_effort: high + service_tier: fast --- You are working on a Linear issue {{ issue.identifier }}. @@ -121,6 +131,12 @@ Notes: - `codex.turn_sandbox_policy` defaults to a `workspaceWrite` policy rooted at the current issue workspace - Supported `codex.approval_policy` values depend on the targeted Codex app-server version. In the current local Codex schema, string values include `untrusted`, `on-failure`, `on-request`, and `never`, and object-form `reject` is also supported. - Supported `codex.thread_sandbox` values: `read-only`, `workspace-write`, `danger-full-access`. +- Optional `codex.model`, `codex.reasoning_effort`, and `codex.service_tier` values are sent on + `thread/start` or `turn/start`, which is useful when `codex.command` is a proxy to an already + running shared app-server and cannot rely on CLI `--config` flags. +- Set `codex.attach_worktree_owner: true` for local Git workspaces to write Codex's + `codex-thread.json` owner metadata after `thread/start`; this lets the Codex app associate the + worktree with the created thread. - When `codex.turn_sandbox_policy` is set explicitly, Symphony passes the map through to Codex unchanged. Compatibility then depends on the targeted Codex app-server version rather than local Symphony validation. @@ -147,7 +163,16 @@ hooks: after_create: | git clone --depth 1 "$SOURCE_REPO_URL" . codex: - command: "$CODEX_BIN --config 'model=\"gpt-5.5\"' app-server" + command: codex app-server proxy --sock "${CODEX_HOME:-$HOME/.codex}/app-server-control/app-server-control.sock" + model: gpt-5.5 + reasoning_effort: xhigh + service_tier: fast + approval_policy: never + thread_sandbox: danger-full-access + turn_sandbox_policy: + type: dangerFullAccess + attach_worktree_owner: true + read_timeout_ms: 60000 ``` - If `WORKFLOW.md` is missing or has invalid YAML at startup, Symphony does not boot. diff --git a/elixir/WORKFLOW.md b/elixir/WORKFLOW.md index 27e82cacc4..9144b85c35 100644 --- a/elixir/WORKFLOW.md +++ b/elixir/WORKFLOW.md @@ -1,7 +1,7 @@ --- tracker: kind: linear - project_slug: "symphony-0c79b11b75ea" + project_slug: "yansen-symphony-b1cf2e8198aa" active_states: - Todo - In Progress @@ -16,24 +16,39 @@ tracker: polling: interval_ms: 5000 workspace: - root: ~/code/symphony-workspaces + root: ~/.codex/worktrees/symphony hooks: + timeout_ms: 900000 after_create: | - git clone --depth 1 https://github.com/openai/symphony . - if command -v mise >/dev/null 2>&1; then - cd elixir && mise trust && mise exec -- mix deps.get + set -eu + repo=/home/dev-user/code/openai + issue="$(basename "$PWD")" + branch="codex/symphony-${issue}" + + git -C "$repo" fetch origin master + git -C "$repo" worktree prune + + if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then + git -C "$repo" worktree add "$PWD" "$branch" + else + git -C "$repo" worktree add -b "$branch" "$PWD" origin/master fi before_remove: | - cd elixir && mise exec -- mix workspace.before_remove + git -C /home/dev-user/code/openai worktree remove --force "$PWD" || true agent: max_concurrent_agents: 10 max_turns: 20 codex: - command: codex --config shell_environment_policy.inherit=all --config 'model="gpt-5.5"' --config model_reasoning_effort=xhigh app-server + command: codex app-server proxy --sock "${CODEX_HOME:-$HOME/.codex}/app-server-control/app-server-control.sock" + model: gpt-5.5 + reasoning_effort: xhigh + service_tier: fast approval_policy: never - thread_sandbox: workspace-write + thread_sandbox: danger-full-access turn_sandbox_policy: - type: workspaceWrite + type: dangerFullAccess + attach_worktree_owner: true + read_timeout_ms: 60000 --- You are working on a Linear ticket `{{ issue.identifier }}` @@ -63,9 +78,10 @@ No description provided. Instructions: -1. This is an unattended orchestration session. Never ask a human to perform follow-up actions. -2. Only stop early for a true blocker (missing required auth/permissions/secrets). If blocked, record it in the workpad and move the issue according to workflow. -3. Final message must report completed actions and blockers only. Do not include "next steps for user". +1. This is an asynchronous orchestration session. Work autonomously, but ask concise questions in Linear when human judgment, missing context, secrets, or permissions are required. +2. Treat new human Linear comments as fresh instructions for the current thread and worktree. Re-read them before continuing after any pause or continuation. +3. Only stop early for a true blocker. If blocked, record the question/blocker in the workpad and leave a clear Linear comment that explains exactly what input is needed. +4. Final message must report completed actions and blockers only. Work only in the provided repository copy. Do not touch any other path. @@ -90,15 +106,16 @@ The agent should be able to talk to Linear, either via a configured Linear MCP s current issue as `related`, and use `blockedBy` when the follow-up depends on the current issue. - Move status only when the matching quality bar is met. -- Operate autonomously end-to-end unless blocked by missing requirements, secrets, or permissions. +- Operate autonomously end-to-end unless blocked by missing requirements, secrets, permissions, or an explicit human product/technical decision. - Use the blocked-access escape hatch only for true external blockers (missing required tools/auth) after exhausting documented fallbacks. +- If a human replies in Linear, treat that reply as authoritative follow-up context for the same Codex thread. Update the workpad before resuming implementation. ## Related skills - `linear`: interact with Linear. - `commit`: produce clean, logical commits during implementation. - `push`: keep remote branch current and publish updates. -- `pull`: keep branch updated with latest `origin/main` before handoff. +- `pull`: keep branch updated with latest `origin/master` before handoff. - `land`: when ticket reaches `Merging`, explicitly open and follow `.codex/skills/land/SKILL.md`, which includes the `land` loop. ## Status map @@ -127,7 +144,7 @@ The agent should be able to talk to Linear, either via a configured Linear MCP s - `Done` -> do nothing and shut down. 4. Check whether a PR already exists for the current branch and whether it is closed. - If a branch PR exists and is `CLOSED` or `MERGED`, treat prior branch work as non-reusable for this run. - - Create a fresh branch from `origin/main` and restart execution flow as a new attempt. + - Create a fresh branch from `origin/master` and restart execution flow as a new attempt. 5. For `Todo` tickets, do startup sequencing in this exact order: - `update_issue(..., state: "In Progress")` - find/create `## Codex Workpad` bootstrap comment @@ -150,7 +167,7 @@ The agent should be able to talk to Linear, either via a configured Linear MCP s 4. Start work by writing/updating a hierarchical plan in the workpad comment. 5. Ensure the workpad includes a compact environment stamp at the top as a code fence line: - Format: `:@` - - Example: `devbox-01:/home/dev-user/code/symphony-workspaces/MT-32@7bdde33bc` + - Example: `devbox-01:/home/dev-user/.codex/worktrees/symphony/MT-32@7bdde33bc` - Do not include metadata already inferable from Linear issue fields (`issue ID`, `status`, `branch`, `PR link`). 6. Add explicit acceptance criteria and TODOs in checklist form in the same comment. - If changes are user-facing, include a UI walkthrough acceptance criterion that describes the end-to-end user path to validate. @@ -158,7 +175,7 @@ The agent should be able to talk to Linear, either via a configured Linear MCP s - If the ticket description/comment context includes `Validation`, `Test Plan`, or `Testing` sections, copy those requirements into the workpad `Acceptance Criteria` and `Validation` sections as required checkboxes (no optional downgrade). 7. Run a principal-style self-review of the plan and refine it in the comment. 8. Before implementing, capture a concrete reproduction signal and record it in the workpad `Notes` section (command/output, screenshot, or deterministic UI behavior). -9. Run the `pull` skill to sync with latest `origin/main` before any code edits, then record the pull/sync result in the workpad `Notes`. +9. Run the `pull` skill to sync with latest `origin/master` before any code edits, then record the pull/sync result in the workpad `Notes`. - Include a `pull skill evidence` note with: - merge source(s), - result (`clean` or `conflicts resolved`), @@ -217,7 +234,7 @@ Use this only when completion is blocked by missing required tools or missing au 7. Before every `git push` attempt, run the required validation for your scope and confirm it passes; if it fails, address issues and rerun until green, then commit and push changes. 8. Attach PR URL to the issue (prefer attachment; use the workpad comment only if attachment is unavailable). - Ensure the GitHub PR has label `symphony` (add it if missing). -9. Merge latest `origin/main` into branch, resolve conflicts, and rerun checks. +9. Merge latest `origin/master` into branch, resolve conflicts, and rerun checks. 10. Update the workpad comment with final checklist status and validation notes. - Mark completed plan/acceptance/validation checklist items as checked. - Add final handoff notes (commit + validation summary) in the same workpad comment. @@ -253,7 +270,7 @@ Use this only when completion is blocked by missing required tools or missing au 2. Re-read the full issue body and all human comments; explicitly identify what will be done differently this attempt. 3. Close the existing PR tied to the issue. 4. Remove the existing `## Codex Workpad` comment from the issue. -5. Create a fresh branch from `origin/main`. +5. Create a fresh branch from `origin/master`. 6. Start over from the normal kickoff flow: - If current issue state is `Todo`, move it to `In Progress`; otherwise keep the current state. - Create a new bootstrap `## Codex Workpad` comment. @@ -272,7 +289,7 @@ Use this only when completion is blocked by missing required tools or missing au ## Guardrails - If the branch PR is already closed/merged, do not reuse that branch or prior implementation state for continuation. -- For closed/merged branch PRs, create a new branch from `origin/main` and restart from reproduction/planning as if starting fresh. +- For closed/merged branch PRs, create a new branch from `origin/master` and restart from reproduction/planning as if starting fresh. - If issue state is `Backlog`, do not modify it; wait for human to move to `Todo`. - Do not edit the issue body/description for planning or progress tracking. - Use exactly one persistent workpad comment (`## Codex Workpad`) per issue. diff --git a/elixir/lib/symphony_elixir/agent_runner.ex b/elixir/lib/symphony_elixir/agent_runner.ex index 502c8cf5f6..52f46db0ce 100644 --- a/elixir/lib/symphony_elixir/agent_runner.ex +++ b/elixir/lib/symphony_elixir/agent_runner.ex @@ -80,7 +80,11 @@ defmodule SymphonyElixir.AgentRunner do max_turns = Keyword.get(opts, :max_turns, Config.settings!().agent.max_turns) issue_state_fetcher = Keyword.get(opts, :issue_state_fetcher, &Tracker.fetch_issue_states_by_ids/1) - with {:ok, session} <- AppServer.start_session(workspace, worker_host: worker_host) do + session_opts = + [worker_host: worker_host] + |> maybe_put_thread_id(Keyword.get(opts, :thread_id)) + + with {:ok, session} <- AppServer.start_session(workspace, session_opts) do try do do_run_codex_turns(session, workspace, issue, codex_update_recipient, opts, issue_state_fetcher, 1, max_turns) after @@ -130,18 +134,23 @@ defmodule SymphonyElixir.AgentRunner do end end - defp build_turn_prompt(issue, opts, 1, _max_turns), do: PromptBuilder.build_prompt(issue, opts) + defp build_turn_prompt(issue, opts, 1, _max_turns) do + human_comments = Keyword.get(opts, :human_comments, []) + + cond do + is_binary(Keyword.get(opts, :prompt)) -> + Keyword.fetch!(opts, :prompt) - defp build_turn_prompt(_issue, _opts, turn_number, max_turns) do - """ - Continuation guidance: + is_list(human_comments) and human_comments != [] -> + PromptBuilder.build_human_followup_prompt(issue, human_comments) + + true -> + PromptBuilder.build_prompt(issue, opts) + end + end - - The previous Codex turn completed normally, but the Linear issue is still in an active state. - - This is continuation turn ##{turn_number} of #{max_turns} for the current agent run. - - Resume from the current workspace and workpad state instead of restarting from scratch. - - The original task instructions and prior turn context are already present in this thread, so do not restate them before acting. - - Focus on the remaining ticket work and do not end the turn while the issue stays active unless you are truly blocked. - """ + defp build_turn_prompt(issue, _opts, turn_number, max_turns) do + PromptBuilder.build_continuation_prompt(issue, turn_number, max_turns) end defp continue_with_issue?(%Issue{id: issue_id} = issue, issue_state_fetcher) when is_binary(issue_id) do @@ -191,6 +200,12 @@ defmodule SymphonyElixir.AgentRunner do defp worker_host_for_log(nil), do: "local" defp worker_host_for_log(worker_host), do: worker_host + defp maybe_put_thread_id(opts, thread_id) when is_binary(thread_id) and thread_id != "" do + Keyword.put(opts, :thread_id, thread_id) + end + + defp maybe_put_thread_id(opts, _thread_id), do: opts + defp normalize_issue_state(state_name) when is_binary(state_name) do state_name |> String.trim() diff --git a/elixir/lib/symphony_elixir/codex/app_server.ex b/elixir/lib/symphony_elixir/codex/app_server.ex index 4b46cbede7..924e832069 100644 --- a/elixir/lib/symphony_elixir/codex/app_server.ex +++ b/elixir/lib/symphony_elixir/codex/app_server.ex @@ -3,19 +3,26 @@ defmodule SymphonyElixir.Codex.AppServer do Minimal client for the Codex app-server JSON-RPC 2.0 stream over stdio. """ + import Bitwise require Logger alias SymphonyElixir.{Codex.DynamicTool, Config, PathSafety, SSH} @initialize_id 1 @thread_start_id 2 @turn_start_id 3 + @thread_archive_id 4 @port_line_bytes 1_048_576 @max_stream_log_bytes 1_000 + @websocket_magic "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + @websocket_handshake_timeout_ms 5_000 @non_interactive_tool_input_answer "This is a non-interactive session. Operator input is unavailable." @type session :: %{ - port: port(), + port: port() | pid(), metadata: map(), + model: String.t() | nil, + reasoning_effort: String.t() | nil, + service_tier: String.t() | nil, approval_policy: String.t() | map(), auto_approve_requests: boolean(), thread_sandbox: String.t(), @@ -39,17 +46,23 @@ defmodule SymphonyElixir.Codex.AppServer do @spec start_session(Path.t(), keyword()) :: {:ok, session()} | {:error, term()} def start_session(workspace, opts \\ []) do worker_host = Keyword.get(opts, :worker_host) + requested_thread_id = Keyword.get(opts, :thread_id) with {:ok, expanded_workspace} <- validate_workspace_cwd(workspace, worker_host), {:ok, port} <- start_port(expanded_workspace, worker_host) do metadata = port_metadata(port, worker_host) with {:ok, session_policies} <- session_policies(expanded_workspace, worker_host), - {:ok, thread_id} <- do_start_session(port, expanded_workspace, session_policies) do + session_policies = Map.put(session_policies, :worker_host, worker_host), + {:ok, thread_id} <- + do_start_session(port, expanded_workspace, session_policies, requested_thread_id) do {:ok, %{ port: port, metadata: metadata, + model: session_policies.model, + reasoning_effort: session_policies.reasoning_effort, + service_tier: session_policies.service_tier, approval_policy: session_policies.approval_policy, auto_approve_requests: session_policies.approval_policy == "never", thread_sandbox: session_policies.thread_sandbox, @@ -71,6 +84,9 @@ defmodule SymphonyElixir.Codex.AppServer do %{ port: port, metadata: metadata, + model: model, + reasoning_effort: reasoning_effort, + service_tier: service_tier, approval_policy: approval_policy, auto_approve_requests: auto_approve_requests, turn_sandbox_policy: turn_sandbox_policy, @@ -88,7 +104,13 @@ defmodule SymphonyElixir.Codex.AppServer do DynamicTool.execute(tool, arguments) end) - case start_turn(port, thread_id, prompt, issue, workspace, approval_policy, turn_sandbox_policy) do + case start_turn(port, thread_id, prompt, issue, workspace, %{ + approval_policy: approval_policy, + turn_sandbox_policy: turn_sandbox_policy, + model: model, + reasoning_effort: reasoning_effort, + service_tier: service_tier + }) do {:ok, turn_id} -> session_id = "#{thread_id}-#{turn_id}" Logger.info("Codex session started for #{issue_context(issue)} session_id=#{session_id}") @@ -140,10 +162,46 @@ defmodule SymphonyElixir.Codex.AppServer do end @spec stop_session(session()) :: :ok - def stop_session(%{port: port}) when is_port(port) do + def stop_session(%{port: port}) do stop_port(port) end + @spec archive_thread(String.t(), keyword()) :: :ok | {:error, term()} + def archive_thread(thread_id, opts \\ []) when is_binary(thread_id) do + worker_host = Keyword.get(opts, :worker_host) + + with :ok <- validate_thread_id(thread_id), + {:ok, workspace} <- archive_workspace(opts), + {:ok, expanded_workspace} <- validate_workspace_cwd(workspace, worker_host), + {:ok, port} <- start_port(expanded_workspace, worker_host) do + try do + with :ok <- send_initialize(port) do + do_archive_thread(port, thread_id) + end + after + stop_port(port) + end + end + end + + defp validate_thread_id(thread_id) when is_binary(thread_id) do + if String.trim(thread_id) == "" do + {:error, :invalid_thread_id} + else + :ok + end + end + + defp archive_workspace(opts) do + case Keyword.get(opts, :workspace) do + workspace when is_binary(workspace) -> + {:ok, workspace} + + _ -> + {:error, :missing_archive_workspace} + end + end + defp validate_workspace_cwd(workspace, nil) when is_binary(workspace) do expanded_workspace = Path.expand(workspace) expanded_root = Path.expand(Config.settings!().workspace.root) @@ -187,6 +245,21 @@ defmodule SymphonyElixir.Codex.AppServer do end defp start_port(workspace, nil) do + command = Config.settings!().codex.command + + case websocket_proxy_socket_path(command) do + {:ok, socket_path} -> start_websocket_proxy(socket_path) + :not_proxy -> start_stdio_port(workspace, command) + {:error, reason} -> {:error, reason} + end + end + + defp start_port(workspace, worker_host) when is_binary(worker_host) do + remote_command = remote_launch_command(workspace) + SSH.start_port(worker_host, remote_command, line: @port_line_bytes) + end + + defp start_stdio_port(workspace, command) do executable = System.find_executable("bash") if is_nil(executable) do @@ -199,7 +272,7 @@ defmodule SymphonyElixir.Codex.AppServer do :binary, :exit_status, :stderr_to_stdout, - args: [~c"-lc", String.to_charlist(Config.settings!().codex.command)], + args: [~c"-lc", String.to_charlist(command)], cd: String.to_charlist(workspace), line: @port_line_bytes ] @@ -209,9 +282,39 @@ defmodule SymphonyElixir.Codex.AppServer do end end - defp start_port(workspace, worker_host) when is_binary(worker_host) do - remote_command = remote_launch_command(workspace) - SSH.start_port(worker_host, remote_command, line: @port_line_bytes) + defp start_websocket_proxy(socket_path) do + expanded_socket_path = expand_proxy_socket_path(socket_path) + + with {:ok, socket} <- + :gen_tcp.connect({:local, expanded_socket_path}, 0, [ + :binary, + active: false, + packet: :raw + ]), + {:ok, initial_buffer} <- websocket_handshake(socket) do + owner = self() + + pid = + spawn_link(fn -> + receive do + {:start_websocket_proxy, socket, initial_buffer} -> + :inet.setopts(socket, active: :once) + websocket_proxy_loop(owner, socket, initial_buffer) + end + end) + + case :gen_tcp.controlling_process(socket, pid) do + :ok -> + send(pid, {:start_websocket_proxy, socket, initial_buffer}) + {:ok, pid} + + {:error, reason} -> + :gen_tcp.close(socket) + {:error, {:websocket_proxy_owner_transfer_failed, reason}} + end + else + {:error, reason} -> {:error, {:websocket_proxy_connect_failed, expanded_socket_path, reason}} + end end defp remote_launch_command(workspace) when is_binary(workspace) do @@ -238,6 +341,250 @@ defmodule SymphonyElixir.Codex.AppServer do end end + defp port_metadata(port, worker_host) when is_pid(port) do + base_metadata = %{codex_app_server_transport: "websocket-proxy"} + + case worker_host do + host when is_binary(host) -> Map.put(base_metadata, :worker_host, host) + _ -> base_metadata + end + end + + defp websocket_proxy_socket_path(command) when is_binary(command) do + if Regex.match?(~r/\bapp-server\s+proxy\b/, command), + do: parse_websocket_proxy_socket_path(command), + else: :not_proxy + end + + defp parse_websocket_proxy_socket_path(command) do + case Regex.run(~r/--sock(?:=|\s+)(?:"([^"]+)"|'([^']+)'|(\S+))/, command, capture: :all_but_first) do + captures when is_list(captures) -> + captures + |> Enum.find(&(&1 != "")) + |> websocket_proxy_socket_result() + + _ -> + {:error, :missing_websocket_proxy_socket} + end + end + + defp websocket_proxy_socket_result(socket_path) when is_binary(socket_path) do + {:ok, socket_path} + end + + defp websocket_proxy_socket_result(_other) do + {:error, :missing_websocket_proxy_socket} + end + + defp expand_proxy_socket_path(path) when is_binary(path) do + codex_home = System.get_env("CODEX_HOME") || Path.join(System.user_home!(), ".codex") + + path + |> String.replace("${CODEX_HOME:-$HOME/.codex}", codex_home) + |> String.replace("${CODEX_HOME}", codex_home) + |> String.replace("$CODEX_HOME", codex_home) + |> String.replace("${HOME}", System.user_home!()) + |> String.replace("$HOME", System.user_home!()) + |> expand_home_path() + |> Path.expand() + end + + defp expand_home_path("~/" <> rest), do: Path.join(System.user_home!(), rest) + defp expand_home_path("~"), do: System.user_home!() + defp expand_home_path(path), do: path + + defp websocket_handshake(socket) do + key = Base.encode64(:crypto.strong_rand_bytes(16)) + + request = [ + "GET /rpc HTTP/1.1\r\n", + "Host: codex-app-server\r\n", + "Upgrade: websocket\r\n", + "Connection: Upgrade\r\n", + "Sec-WebSocket-Key: ", + key, + "\r\n", + "Sec-WebSocket-Version: 13\r\n", + "\r\n" + ] + + with :ok <- :gen_tcp.send(socket, request), + {:ok, response, initial_buffer} <- + websocket_read_http_response(socket, "", handshake_deadline()), + :ok <- validate_websocket_handshake(response, key) do + {:ok, initial_buffer} + end + end + + defp handshake_deadline do + System.monotonic_time(:millisecond) + @websocket_handshake_timeout_ms + end + + defp websocket_read_http_response(socket, buffer, deadline_ms) do + case :binary.match(buffer, "\r\n\r\n") do + {header_end, 4} -> + response = binary_part(buffer, 0, header_end + 4) + rest = binary_part(buffer, header_end + 4, byte_size(buffer) - header_end - 4) + {:ok, response, rest} + + :nomatch -> + timeout_ms = max(deadline_ms - System.monotonic_time(:millisecond), 0) + + case :gen_tcp.recv(socket, 0, timeout_ms) do + {:ok, chunk} -> + websocket_read_http_response(socket, buffer <> chunk, deadline_ms) + + {:error, :timeout} -> + {:error, :websocket_handshake_timeout} + + {:error, reason} -> + {:error, {:websocket_handshake_recv_failed, reason}} + end + end + end + + defp validate_websocket_handshake(response, key) do + [status_line | header_lines] = String.split(response, "\r\n", trim: true) + + headers = + Enum.reduce(header_lines, %{}, fn line, acc -> + case String.split(line, ":", parts: 2) do + [name, value] -> Map.put(acc, String.downcase(String.trim(name)), String.trim(value)) + _ -> acc + end + end) + + expected_accept = + :crypto.hash(:sha, key <> @websocket_magic) + |> Base.encode64() + + cond do + not String.contains?(status_line, " 101 ") -> + {:error, {:websocket_handshake_rejected, status_line}} + + Map.get(headers, "sec-websocket-accept") != expected_accept -> + {:error, :invalid_websocket_accept} + + true -> + :ok + end + end + + defp websocket_proxy_loop(owner, socket, buffer) do + case websocket_deliver_frames(owner, socket, buffer) do + {:ok, remaining_buffer} -> + receive do + {:send, payload} when is_binary(payload) -> + _ = :gen_tcp.send(socket, websocket_client_frame(payload, 0x1)) + websocket_proxy_loop(owner, socket, remaining_buffer) + + :stop -> + _ = :gen_tcp.send(socket, websocket_client_frame("", 0x8)) + :gen_tcp.close(socket) + + {:tcp, ^socket, chunk} -> + :inet.setopts(socket, active: :once) + websocket_proxy_loop(owner, socket, remaining_buffer <> chunk) + + {:tcp_closed, ^socket} -> + send(owner, {self(), {:exit_status, 0}}) + + {:tcp_error, ^socket, reason} -> + send(owner, {self(), {:exit_status, {:tcp_error, reason}}}) + end + + :closed -> + :ok + end + end + + defp websocket_deliver_frames(owner, socket, buffer) do + case websocket_decode_frame(buffer) do + {:ok, %{opcode: opcode, payload: payload}, rest} when opcode in [0x1, 0x2] -> + send(owner, {self(), {:data, {:eol, payload}}}) + websocket_deliver_frames(owner, socket, rest) + + {:ok, %{opcode: 0x8}, _rest} -> + send(owner, {self(), {:exit_status, 0}}) + :gen_tcp.close(socket) + :closed + + {:ok, %{opcode: 0x9, payload: payload}, rest} -> + _ = :gen_tcp.send(socket, websocket_client_frame(payload, 0xA)) + websocket_deliver_frames(owner, socket, rest) + + {:ok, _frame, rest} -> + websocket_deliver_frames(owner, socket, rest) + + :more -> + {:ok, buffer} + end + end + + defp websocket_decode_frame(buffer) when byte_size(buffer) < 2, do: :more + + defp websocket_decode_frame(<>) do + opcode = first &&& 0x0F + masked? = (second &&& 0x80) == 0x80 + length_code = second &&& 0x7F + + with {:ok, length, rest_after_length} <- websocket_payload_length(length_code, rest), + {:ok, mask_key, rest_after_mask} <- websocket_mask_key(masked?, rest_after_length), + true <- byte_size(rest_after_mask) >= length do + <> = rest_after_mask + + payload = + if masked? do + websocket_mask_payload(payload, mask_key) + else + payload + end + + {:ok, %{opcode: opcode, payload: payload}, remaining} + else + false -> :more + :more -> :more + end + end + + defp websocket_payload_length(length, rest) when length < 126, do: {:ok, length, rest} + + defp websocket_payload_length(126, <>), do: {:ok, length, rest} + defp websocket_payload_length(126, _rest), do: :more + + defp websocket_payload_length(127, <>), do: {:ok, length, rest} + defp websocket_payload_length(127, _rest), do: :more + + defp websocket_mask_key(false, rest), do: {:ok, nil, rest} + defp websocket_mask_key(true, <>), do: {:ok, mask_key, rest} + defp websocket_mask_key(true, _rest), do: :more + + defp websocket_client_frame(payload, opcode) when is_binary(payload) do + mask_key = :crypto.strong_rand_bytes(4) + length = byte_size(payload) + + length_header = + cond do + length < 126 -> <<0x80 ||| length>> + length <= 65_535 -> <<0x80 ||| 126, length::16>> + true -> <<0x80 ||| 127, length::64>> + end + + <<0x80 ||| opcode>> <> length_header <> mask_key <> websocket_mask_payload(payload, mask_key) + end + + defp websocket_mask_payload(payload, nil), do: payload + + defp websocket_mask_payload(payload, <>) do + mask = {a, b, c, d} + + payload + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.map(fn {byte, index} -> bxor(byte, elem(mask, rem(index, 4))) end) + |> :binary.list_to_bin() + end + defp send_initialize(port) do payload = %{ "method" => "initialize", @@ -270,42 +617,72 @@ defmodule SymphonyElixir.Codex.AppServer do Config.codex_runtime_settings(workspace, remote: true) end - defp do_start_session(port, workspace, session_policies) do + defp do_start_session(port, workspace, session_policies, nil) do case send_initialize(port) do :ok -> start_thread(port, workspace, session_policies) {:error, reason} -> {:error, reason} end end - defp start_thread(port, workspace, %{approval_policy: approval_policy, thread_sandbox: thread_sandbox}) do + defp do_start_session(port, workspace, session_policies, thread_id) when is_binary(thread_id) do + with :ok <- validate_thread_id(thread_id), + :ok <- send_initialize(port), + :ok <- maybe_attach_worktree_owner(workspace, thread_id, session_policies) do + {:ok, thread_id} + end + end + + defp do_start_session(_port, _workspace, _session_policies, thread_id), do: {:error, {:invalid_thread_id, thread_id}} + + defp do_archive_thread(port, thread_id) do send_message(port, %{ - "method" => "thread/start", - "id" => @thread_start_id, + "method" => "thread/archive", + "id" => @thread_archive_id, "params" => %{ - "approvalPolicy" => approval_policy, - "sandbox" => thread_sandbox, + "threadId" => thread_id + } + }) + + case await_response(port, @thread_archive_id) do + {:ok, _result} -> :ok + other -> other + end + end + + defp start_thread(port, workspace, session_policies) do + params = + %{ + "approvalPolicy" => session_policies.approval_policy, + "sandbox" => session_policies.thread_sandbox, "cwd" => workspace, "dynamicTools" => DynamicTool.tool_specs() } + |> maybe_put("model", session_policies.model) + |> maybe_put("serviceTier", session_policies.service_tier) + + send_message(port, %{ + "method" => "thread/start", + "id" => @thread_start_id, + "params" => params }) case await_response(port, @thread_start_id) do - {:ok, %{"thread" => thread_payload}} -> - case thread_payload do - %{"id" => thread_id} -> {:ok, thread_id} - _ -> {:error, {:invalid_thread_payload, thread_payload}} + {:ok, %{"thread" => %{"id" => thread_id}}} -> + with :ok <- maybe_attach_worktree_owner(workspace, thread_id, session_policies) do + {:ok, thread_id} end + {:ok, %{"thread" => thread_payload}} -> + {:error, {:invalid_thread_payload, thread_payload}} + other -> other end end - defp start_turn(port, thread_id, prompt, issue, workspace, approval_policy, turn_sandbox_policy) do - send_message(port, %{ - "method" => "turn/start", - "id" => @turn_start_id, - "params" => %{ + defp start_turn(port, thread_id, prompt, issue, workspace, session_policies) do + params = + %{ "threadId" => thread_id, "input" => [ %{ @@ -315,9 +692,18 @@ defmodule SymphonyElixir.Codex.AppServer do ], "cwd" => workspace, "title" => "#{issue.identifier}: #{issue.title}", - "approvalPolicy" => approval_policy, - "sandboxPolicy" => turn_sandbox_policy + "approvalPolicy" => session_policies.approval_policy, + "sandboxPolicy" => session_policies.turn_sandbox_policy } + |> maybe_put("model", session_policies.model) + |> maybe_put("effort", session_policies.reasoning_effort) + |> maybe_put("serviceTier", session_policies.service_tier) + |> maybe_put("collaborationMode", collaboration_mode(session_policies)) + + send_message(port, %{ + "method" => "turn/start", + "id" => @turn_start_id, + "params" => params }) case await_response(port, @turn_start_id) do @@ -326,6 +712,68 @@ defmodule SymphonyElixir.Codex.AppServer do end end + defp maybe_attach_worktree_owner(_workspace, _thread_id, %{attach_worktree_owner: false}), do: :ok + + defp maybe_attach_worktree_owner(_workspace, _thread_id, %{attach_worktree_owner: true, worker_host: worker_host}) + when is_binary(worker_host) do + {:error, {:worktree_owner_attach_failed, {:unsupported_remote_worker, worker_host}}} + end + + defp maybe_attach_worktree_owner(workspace, thread_id, %{attach_worktree_owner: true}) do + with :ok <- validate_thread_id(thread_id), + {:ok, git_path} <- git_thread_config_path(workspace), + :ok <- File.mkdir_p(Path.dirname(git_path)), + {:ok, payload} <- Jason.encode(%{version: 1, ownerThreadId: thread_id}), + :ok <- File.write(git_path, payload <> "\n") do + :ok + else + {:error, reason} -> {:error, {:worktree_owner_attach_failed, reason}} + end + end + + defp git_thread_config_path(workspace) do + with git when is_binary(git) <- System.find_executable("git"), + {output, 0} <- + System.cmd(git, ["-C", workspace, "rev-parse", "--git-path", "codex-thread.json"], stderr_to_stdout: true) do + output + |> String.trim() + |> git_thread_config_result(workspace) + else + nil -> {:error, :git_not_found} + {output, status} -> {:error, {:git_rev_parse_failed, status, String.trim(output)}} + end + end + + defp git_thread_config_result("", _workspace), do: {:error, :empty_git_path} + + defp git_thread_config_result(path, workspace) do + case Path.type(path) do + :absolute -> {:ok, path} + _relative -> {:ok, Path.expand(path, workspace)} + end + end + + defp collaboration_mode(%{model: model, reasoning_effort: reasoning_effort}) + when is_binary(model) and model != "" do + %{ + "mode" => "default", + "settings" => %{ + "model" => model, + "reasoning_effort" => normalize_optional_string(reasoning_effort), + "developer_instructions" => nil + } + } + end + + defp collaboration_mode(_session_policies), do: nil + + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, _key, ""), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) + + defp normalize_optional_string(value) when is_binary(value) and value != "", do: value + defp normalize_optional_string(_value), do: nil + defp await_turn_completion(port, on_message, tool_executor, auto_approve_requests) do receive_loop( port, @@ -1006,6 +1454,14 @@ defmodule SymphonyElixir.Codex.AppServer do end end + defp stop_port(port) when is_pid(port) do + if Process.alive?(port) do + send(port, :stop) + end + + :ok + end + defp emit_message(on_message, event, details, metadata) when is_function(on_message, 1) do message = metadata |> Map.merge(details) |> Map.put(:event, event) |> Map.put(:timestamp, DateTime.utc_now()) on_message.(message) @@ -1055,8 +1511,12 @@ defmodule SymphonyElixir.Codex.AppServer do defp tool_call_arguments(_params), do: %{} defp send_message(port, message) do - line = Jason.encode!(message) <> "\n" - Port.command(port, line) + payload = Jason.encode!(message) + + case port do + port when is_port(port) -> Port.command(port, payload <> "\n") + port when is_pid(port) -> send(port, {:send, payload}) + end end defp needs_input?("mcpServer/elicitation/request", payload) when is_map(payload), do: true diff --git a/elixir/lib/symphony_elixir/config.ex b/elixir/lib/symphony_elixir/config.ex index 00e7f9b7ef..df086c931e 100644 --- a/elixir/lib/symphony_elixir/config.ex +++ b/elixir/lib/symphony_elixir/config.ex @@ -21,9 +21,13 @@ defmodule SymphonyElixir.Config do """ @type codex_runtime_settings :: %{ + model: String.t() | nil, + reasoning_effort: String.t() | nil, + service_tier: String.t() | nil, approval_policy: String.t() | map(), thread_sandbox: String.t(), - turn_sandbox_policy: map() + turn_sandbox_policy: map(), + attach_worktree_owner: boolean() } @spec settings() :: {:ok, Schema.t()} | {:error, term()} @@ -106,9 +110,13 @@ defmodule SymphonyElixir.Config do Schema.resolve_runtime_turn_sandbox_policy(settings, workspace, opts) do {:ok, %{ + model: settings.codex.model, + reasoning_effort: settings.codex.reasoning_effort, + service_tier: settings.codex.service_tier, approval_policy: settings.codex.approval_policy, thread_sandbox: settings.codex.thread_sandbox, - turn_sandbox_policy: turn_sandbox_policy + turn_sandbox_policy: turn_sandbox_policy, + attach_worktree_owner: settings.codex.attach_worktree_owner }} end end diff --git a/elixir/lib/symphony_elixir/config/schema.ex b/elixir/lib/symphony_elixir/config/schema.ex index 17ead4ad6b..8a2af76ba7 100644 --- a/elixir/lib/symphony_elixir/config/schema.ex +++ b/elixir/lib/symphony_elixir/config/schema.ex @@ -158,6 +158,9 @@ defmodule SymphonyElixir.Config.Schema do @primary_key false embedded_schema do field(:command, :string, default: "codex app-server") + field(:model, :string) + field(:reasoning_effort, :string) + field(:service_tier, :string) field(:approval_policy, StringOrMap, default: %{ @@ -171,6 +174,7 @@ defmodule SymphonyElixir.Config.Schema do field(:thread_sandbox, :string, default: "workspace-write") field(:turn_sandbox_policy, :map) + field(:attach_worktree_owner, :boolean, default: false) field(:turn_timeout_ms, :integer, default: 3_600_000) field(:read_timeout_ms, :integer, default: 5_000) field(:stall_timeout_ms, :integer, default: 300_000) @@ -183,9 +187,13 @@ defmodule SymphonyElixir.Config.Schema do attrs, [ :command, + :model, + :reasoning_effort, + :service_tier, :approval_policy, :thread_sandbox, :turn_sandbox_policy, + :attach_worktree_owner, :turn_timeout_ms, :read_timeout_ms, :stall_timeout_ms diff --git a/elixir/lib/symphony_elixir/linear/client.ex b/elixir/lib/symphony_elixir/linear/client.ex index 1a19649660..ff39debd84 100644 --- a/elixir/lib/symphony_elixir/linear/client.ex +++ b/elixir/lib/symphony_elixir/linear/client.ex @@ -4,13 +4,14 @@ defmodule SymphonyElixir.Linear.Client do """ require Logger - alias SymphonyElixir.{Config, Linear.Issue} + alias SymphonyElixir.Config + alias SymphonyElixir.Linear.{Comment, Issue} @issue_page_size 50 @max_error_body_log_bytes 1_000 @query """ - query SymphonyLinearPoll($projectSlug: String!, $stateNames: [String!]!, $first: Int!, $relationFirst: Int!, $after: String) { + query SymphonyLinearPoll($projectSlug: String!, $stateNames: [String!]!, $first: Int!, $relationFirst: Int!, $commentFirst: Int!, $after: String) { issues(filter: {project: {slugId: {eq: $projectSlug}}, state: {name: {in: $stateNames}}}, first: $first, after: $after) { nodes { id @@ -43,6 +44,18 @@ defmodule SymphonyElixir.Linear.Client do } } } + comments(first: $commentFirst) { + nodes { + id + body + createdAt + updatedAt + user { + id + name + } + } + } createdAt updatedAt } @@ -55,7 +68,7 @@ defmodule SymphonyElixir.Linear.Client do """ @query_by_ids """ - query SymphonyLinearIssuesById($ids: [ID!]!, $first: Int!, $relationFirst: Int!) { + query SymphonyLinearIssuesById($ids: [ID!]!, $first: Int!, $relationFirst: Int!, $commentFirst: Int!) { issues(filter: {id: {in: $ids}}, first: $first) { nodes { id @@ -88,6 +101,18 @@ defmodule SymphonyElixir.Linear.Client do } } } + comments(first: $commentFirst) { + nodes { + id + body + createdAt + updatedAt + user { + id + name + } + } + } createdAt updatedAt } @@ -247,6 +272,7 @@ defmodule SymphonyElixir.Linear.Client do stateNames: state_names, first: @issue_page_size, relationFirst: @issue_page_size, + commentFirst: @issue_page_size, after: after_cursor }), {:ok, issues, page_info} <- decode_linear_page_response(body, assignee_filter) do @@ -294,7 +320,8 @@ defmodule SymphonyElixir.Linear.Client do case graphql_fun.(@query_by_ids, %{ ids: batch_ids, first: length(batch_ids), - relationFirst: @issue_page_size + relationFirst: @issue_page_size, + commentFirst: @issue_page_size }) do {:ok, body} -> with {:ok, issues} <- decode_linear_response(body, assignee_filter) do @@ -459,6 +486,7 @@ defmodule SymphonyElixir.Linear.Client do url: issue["url"], assignee_id: assignee_field(assignee, "id"), blocked_by: extract_blockers(issue), + comments: extract_comments(issue), labels: extract_labels(issue), assigned_to_worker: assigned_to_worker?(assignee, assignee_filter), created_at: parse_datetime(issue["createdAt"]), @@ -547,6 +575,32 @@ defmodule SymphonyElixir.Linear.Client do defp extract_labels(_), do: [] + defp extract_comments(%{"comments" => %{"nodes" => comments}}) when is_list(comments) do + comments + |> Enum.map(&normalize_comment/1) + |> Enum.reject(&is_nil/1) + |> Enum.sort_by(fn %Comment{} = comment -> + {comment.created_at && DateTime.to_unix(comment.created_at, :microsecond), comment.id || ""} + end) + end + + defp extract_comments(_), do: [] + + defp normalize_comment(comment) when is_map(comment) do + user = comment["user"] + + %Comment{ + id: comment["id"], + body: comment["body"], + author_id: assignee_field(user, "id"), + author_name: assignee_field(user, "name"), + created_at: parse_datetime(comment["createdAt"]), + updated_at: parse_datetime(comment["updatedAt"]) + } + end + + defp normalize_comment(_comment), do: nil + defp extract_blockers(%{"inverseRelations" => %{"nodes" => inverse_relations}}) when is_list(inverse_relations) do inverse_relations diff --git a/elixir/lib/symphony_elixir/linear/comment.ex b/elixir/lib/symphony_elixir/linear/comment.ex new file mode 100644 index 0000000000..c1389d7da0 --- /dev/null +++ b/elixir/lib/symphony_elixir/linear/comment.ex @@ -0,0 +1,116 @@ +defmodule SymphonyElixir.Linear.Comment do + @moduledoc """ + Normalized Linear comment representation. + """ + + defstruct [ + :id, + :body, + :author_id, + :author_name, + :created_at, + :updated_at + ] + + @type cursor :: %{created_at: DateTime.t() | nil, id: String.t() | nil} + + @type t :: %__MODULE__{ + id: String.t() | nil, + body: String.t() | nil, + author_id: String.t() | nil, + author_name: String.t() | nil, + created_at: DateTime.t() | nil, + updated_at: DateTime.t() | nil + } + + @ignored_markers [ + " + ## Codex needs input + + Symphony paused this issue because the Codex thread needs human input. + #{thread_line} + Reason: #{error} + Last event: #{last_message} + + Reply in this Linear issue to resume the same Codex thread and worktree. + """ + |> String.trim() + end + defp select_worker_host(%State{} = state, preferred_worker_host) do case Config.settings!().worker.ssh_hosts do [] -> @@ -1298,6 +1529,11 @@ defmodule SymphonyElixir.Orchestrator do defp running_entry_session_id(_running_entry), do: "n/a" + defp running_entry_thread_id(%{thread_id: thread_id}) when is_binary(thread_id), + do: thread_id + + defp running_entry_thread_id(_running_entry), do: nil + defp issue_context(%Issue{id: issue_id, identifier: identifier}) do "issue_id=#{issue_id} issue_identifier=#{identifier}" end @@ -1357,6 +1593,7 @@ defmodule SymphonyElixir.Orchestrator do worker_host: Map.get(metadata, :worker_host), workspace_path: Map.get(metadata, :workspace_path), session_id: metadata.session_id, + thread_id: Map.get(metadata, :thread_id), codex_app_server_pid: metadata.codex_app_server_pid, codex_input_tokens: metadata.codex_input_tokens, codex_output_tokens: metadata.codex_output_tokens, @@ -1380,7 +1617,8 @@ defmodule SymphonyElixir.Orchestrator do identifier: Map.get(retry, :identifier), error: Map.get(retry, :error), worker_host: Map.get(retry, :worker_host), - workspace_path: Map.get(retry, :workspace_path) + workspace_path: Map.get(retry, :workspace_path), + thread_id: Map.get(retry, :thread_id) } end) @@ -1394,6 +1632,7 @@ defmodule SymphonyElixir.Orchestrator do worker_host: Map.get(metadata, :worker_host), workspace_path: Map.get(metadata, :workspace_path), session_id: Map.get(metadata, :session_id), + thread_id: Map.get(metadata, :thread_id), error: Map.get(metadata, :error), blocked_at: Map.get(metadata, :blocked_at), last_codex_timestamp: Map.get(metadata, :last_codex_timestamp), @@ -1451,6 +1690,7 @@ defmodule SymphonyElixir.Orchestrator do last_codex_timestamp: timestamp, last_codex_message: summarize_codex_update(update), session_id: session_id_for_update(running_entry.session_id, update), + thread_id: thread_id_for_update(Map.get(running_entry, :thread_id), update), last_codex_event: event, codex_app_server_pid: codex_app_server_pid_for_update(codex_app_server_pid, update), codex_input_tokens: codex_input_tokens + token_delta.input_tokens, @@ -1483,6 +1723,11 @@ defmodule SymphonyElixir.Orchestrator do defp session_id_for_update(existing, _update), do: existing + defp thread_id_for_update(_existing, %{thread_id: thread_id}) when is_binary(thread_id), + do: thread_id + + defp thread_id_for_update(existing, _update), do: existing + defp turn_count_for_update(existing_count, existing_session_id, %{ event: :session_started, session_id: session_id diff --git a/elixir/lib/symphony_elixir/prompt_builder.ex b/elixir/lib/symphony_elixir/prompt_builder.ex index 1b334a105d..26cc7aa0e1 100644 --- a/elixir/lib/symphony_elixir/prompt_builder.ex +++ b/elixir/lib/symphony_elixir/prompt_builder.ex @@ -3,7 +3,9 @@ defmodule SymphonyElixir.PromptBuilder do Builds agent prompts from Linear issue data. """ - alias SymphonyElixir.{Config, Workflow} + alias SymphonyElixir.Config + alias SymphonyElixir.Linear.Comment + alias SymphonyElixir.Workflow @render_opts [strict_variables: true, strict_filters: true] @@ -25,6 +27,33 @@ defmodule SymphonyElixir.PromptBuilder do |> IO.iodata_to_binary() end + @spec build_continuation_prompt(SymphonyElixir.Linear.Issue.t(), pos_integer(), pos_integer()) :: String.t() + def build_continuation_prompt(issue, turn_number, max_turns) do + """ + Continuation guidance: + + - The previous Codex turn completed normally, but the Linear issue is still in an active state. + - This is continuation turn ##{turn_number} of #{max_turns} for the current agent run. + - Resume from the current workspace and workpad state instead of restarting from scratch. + - The original task instructions and prior turn context are already present in this thread, so do not restate them before acting. + - Re-read the Linear issue, the workpad, and any recent human comments before choosing the next action. + - Focus on the remaining ticket work and do not end the turn while the issue stays active unless you are truly blocked. + + #{human_comment_section(issue)} + """ + end + + @spec build_human_followup_prompt(SymphonyElixir.Linear.Issue.t(), [Comment.t()]) :: String.t() + def build_human_followup_prompt(issue, comments) when is_list(comments) do + """ + Human Linear follow-up for #{issue.identifier || issue.id || "this issue"}: + + The user added the Linear comment(s) below. Treat them as fresh human input for this same Codex thread and workspace. If they answer a blocker, continue the work. If they redirect or correct the implementation, update the plan/workpad and act on the new direction. + + #{format_comments(comments)} + """ + end + defp prompt_template!({:ok, %{prompt_template: prompt}}), do: default_prompt(prompt) defp prompt_template!({:error, reason}) do @@ -61,4 +90,38 @@ defmodule SymphonyElixir.PromptBuilder do prompt end end + + defp human_comment_section(%{comments: comments}) when is_list(comments) do + comments + |> Enum.filter(&Comment.human_feedback?/1) + |> case do + [] -> + "" + + human_comments -> + """ + Recent human Linear comments: + + #{format_comments(human_comments)} + """ + end + end + + defp human_comment_section(_issue), do: "" + + defp format_comments(comments) do + comments + |> Enum.filter(&Comment.human_feedback?/1) + |> Enum.map_join("\n\n", fn %Comment{} = comment -> + """ + From #{Comment.author_label(comment)} at #{Comment.timestamp_label(comment)}: + #{String.trim(comment.body || "")} + """ + |> String.trim() + end) + |> case do + "" -> "No human comments were included." + formatted -> formatted + end + end end diff --git a/elixir/lib/symphony_elixir/workspace.ex b/elixir/lib/symphony_elixir/workspace.ex index 14e29da402..1dd2cd3ee3 100644 --- a/elixir/lib/symphony_elixir/workspace.ex +++ b/elixir/lib/symphony_elixir/workspace.ex @@ -34,7 +34,7 @@ defmodule SymphonyElixir.Workspace do defp ensure_workspace(workspace, nil) do cond do File.dir?(workspace) -> - {:ok, workspace, false} + {:ok, workspace, empty_directory?(workspace)} File.exists?(workspace) -> File.rm_rf!(workspace) @@ -51,7 +51,11 @@ defmodule SymphonyElixir.Workspace do "set -eu", remote_shell_assign("workspace", workspace), "if [ -d \"$workspace\" ]; then", - " created=0", + " if [ -z \"$(find \"$workspace\" -mindepth 1 -maxdepth 1 -print -quit)\" ]; then", + " created=1", + " else", + " created=0", + " fi", "elif [ -e \"$workspace\" ]; then", " rm -rf \"$workspace\"", " mkdir -p \"$workspace\"", @@ -78,6 +82,14 @@ defmodule SymphonyElixir.Workspace do end end + defp empty_directory?(workspace) do + case File.ls(workspace) do + {:ok, []} -> true + {:ok, _entries} -> false + {:error, reason} -> raise File.Error, reason: reason, action: "list directory", path: workspace + end + end + defp create_workspace(workspace) do File.rm_rf!(workspace) File.mkdir_p!(workspace) diff --git a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex index 43fc3ef642..e963f04707 100644 --- a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex +++ b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex @@ -169,6 +169,18 @@ defmodule SymphonyElixirWeb.DashboardLive do
+ <%= if entry.codex_thread_url do %> + Open thread + + <% end %> <%= if entry.session_id do %> + <% end %> <%= if entry.session_id do %> - <% end %> <%= if entry.session_id do %> - <% end %> <%= if entry.session_id do %>