From 6c75ee3f06dd3cc1ebec68c1442bfa01eec31806 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Nov 2025 18:28:35 +0000 Subject: [PATCH 1/3] Initial plan From bfee9dbd2459024df87768b35e96a22c8acff27b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Nov 2025 18:40:47 +0000 Subject: [PATCH 2/3] Fix Logger configuration for Elixir 1.19 and format code Co-authored-by: mikkihugo <17744793+mikkihugo@users.noreply.github.com> --- config/config.exs | 57 +------------------ .../dag/workflow_definition.ex | 26 ++++----- 2 files changed, 14 insertions(+), 69 deletions(-) diff --git a/config/config.exs b/config/config.exs index c5334a6..31eb5a5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -17,62 +17,7 @@ config :singularity_workflow, # Configures Elixir's Logger config :logger, level: :info, - metadata: [ - :action, - :arity, - :attempt, - :attempts, - :batch_size, - :channel, - :coordination, - :count, - :delay_ms, - :duration_ms, - :elapsed_ms, - :error, - :execution_mode, - :expect_reply, - :expected, - :failed_count, - :function, - :gpu_device, - :gpu_info, - :input_keys, - :job_id, - :job_module, - :limit, - :listener_pid, - :max_retries, - :message_id, - :message_type, - :module, - :msg_count, - :operation, - :original_run_id, - :payload, - :pid, - :poll_interval, - :queue, - :reason, - :reset_all, - :resources, - :run_id, - :state, - :step_name, - :step_slug, - :task_count, - :task_index, - :task_timeout_ms, - :timeout, - :timeout_ms, - :value, - :worker_id, - :workflow, - :workflow_module, - :workflow_name, - :workflow_run_id, - :workflow_slug - ] + metadata: :all # Use Jason for JSON parsing config :singularity_workflow, :json_library, Jason diff --git a/lib/singularity_workflow/dag/workflow_definition.ex b/lib/singularity_workflow/dag/workflow_definition.ex index 22ed800..bfbcb18 100644 --- a/lib/singularity_workflow/dag/workflow_definition.ex +++ b/lib/singularity_workflow/dag/workflow_definition.ex @@ -242,19 +242,19 @@ defmodule Singularity.Workflow.DAG.WorkflowDefinition do new_visited = MapSet.put(visited, step) new_path = [step | path] - # Get dependencies (empty list if step has no dependencies) - deps = Map.get(dependencies, step, []) - - # Visit each dependency; if any returns a cycle, propagate it - # All dependencies checked, no cycle found - Enum.find_value(deps, :no_cycle, fn dep -> - case dfs_cycle(dep, dependencies, new_visited, new_path) do - # Cycle found, propagate - {:cycle, _} = result -> result - # No cycle in this branch - :no_cycle -> nil - end - end) || :no_cycle + # Get dependencies (empty list if step has no dependencies) + deps = Map.get(dependencies, step, []) + + # Visit each dependency; if any returns a cycle, propagate it + # All dependencies checked, no cycle found + Enum.find_value(deps, :no_cycle, fn dep -> + case dfs_cycle(dep, dependencies, new_visited, new_path) do + # Cycle found, propagate + {:cycle, _} = result -> result + # No cycle in this branch + :no_cycle -> nil + end + end) || :no_cycle end end From 2add58b91a24636a71db613f39967fada694638c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Nov 2025 18:43:41 +0000 Subject: [PATCH 3/3] Remove unnecessary markdown files and update documentation references Co-authored-by: mikkihugo <17744793+mikkihugo@users.noreply.github.com> --- GETTING_STARTED.md | 463 ------------------------------------------ RELEASING.md | 198 ------------------ docs/ARCHITECTURE.md | 2 +- docs/README.md | 5 +- docs/TESTING_GUIDE.md | 2 +- 5 files changed, 4 insertions(+), 666 deletions(-) delete mode 100644 GETTING_STARTED.md delete mode 100644 RELEASING.md diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md deleted file mode 100644 index 9977c7f..0000000 --- a/GETTING_STARTED.md +++ /dev/null @@ -1,463 +0,0 @@ -# Getting Started with Singularity.Workflow - -> **📦 This is a library** - You add it to your Elixir application as a dependency, just like Ecto or Oban. - -Singularity.Workflow is a **library package** that provides database-driven workflow orchestration for your Elixir applications. This guide walks you through adding it to your application, basic setup, and running your first workflow. - -## Installation - -Add `singularity_workflow` to **your application's** `mix.exs` dependencies: - -```elixir -# In YOUR application's mix.exs -def deps do - [ - {:singularity_workflow, "~> 1.0.0"} - ] -end -``` - -Then run: - -```bash -mix deps.get -``` - -**Important**: This installs the library into your application. You don't run Singularity.Workflow as a standalone service - it's code you use within your app. - -## Database Setup in Your Application - -Singularity.Workflow uses **your application's database** and requires PostgreSQL 12+ with the `pgmq` extension. - -### 1. Your Database (if you don't have one) - -```bash -# Create your application's database -createdb my_app_dev -``` - -### 2. Configure Your Application's Repo - -Singularity.Workflow uses **your existing Ecto repo** - no separate repo needed: - -```elixir -# config/config.exs in YOUR application -config :my_app, MyApp.Repo, - database: "my_app_dev", - username: "postgres", - password: "postgres", - hostname: "localhost" - -config :my_app, - ecto_repos: [MyApp.Repo] -``` - -### 3. Install pgmq Extension - -```bash -# Install pgmq extension in YOUR database -psql my_app_dev -c "CREATE EXTENSION IF NOT EXISTS pgmq" -``` - -### 4. Run Migrations (Optional) - -The library includes migrations for workflow tables. You can copy them to your app if needed: - -```bash -# Singularity.Workflow tables will be created automatically -# when you execute workflows using your repo -``` - -The library manages these tables: -- `workflow_runs` - Tracks workflow execution instances -- `workflow_step_states` - State for each step in a run -- `workflow_step_tasks` - Individual tasks for map steps -- `workflow_step_dependencies` - Dependency edges in the DAG -- pgmq queue tables - For task coordination - -## Your First Workflow - -### 1. Define a Workflow in Your Application - -Create a workflow module in your application that uses the library: - -```elixir -# In YOUR application: lib/my_app/workflows/hello_world.ex -defmodule MyApp.Workflows.HelloWorld do - def __workflow_steps__ do - [ - {:greet, &__MODULE__.greet/1, depends_on: []} - ] - end - - def greet(_input) do - {:ok, %{"message" => "Hello, World!"}} - end - - @impl true - def on_complete(run_id, output, context) do - IO.inspect(output, label: "Workflow completed") - :ok - end - - @impl true - def on_failure(run_id, error, context) do - IO.inspect(error, label: "Workflow failed") - :ok - end -end -``` - -### 2. Start the Workflow - -```elixir -alias MyApp.Workflows.HelloWorld - -# Start a new workflow run -{:ok, run_id} = HelloWorld.start(%{"name" => "Alice"}) - -# Check status -{:ok, run} = HelloWorld.status(run_id) -IO.inspect(run) -# => %Singularity.Workflow.WorkflowRun{ -# id: "...", -# workflow_slug: "MyApp.Workflows.HelloWorld", -# status: "started", -# input: %{"name" => "Alice"}, -# remaining_steps: 1 -# } -``` - -### 3. Execute Pending Tasks - -The workflow engine coordinates task execution via the pgmq queue. To process tasks: - -```elixir -alias Singularity.Workflow.Executor - -# Poll the queue and execute pending tasks -{:ok, executed_count} = Executor.execute_pending_tasks() - -# Wait for completion -:timer.sleep(1000) - -# Check final status -{:ok, run} = HelloWorld.status(run_id) -IO.inspect(run.status) # => "completed" -``` - -## DAG Workflows with Dependencies - -Singularity.Workflow supports complex DAG workflows with parallel execution and dependency management: - -```elixir -defmodule MyApp.Workflows.DataPipeline do - @behaviour Singularity.Workflow.Executor.Workflow - - @impl true - def definition do - %{ - "version" => "1.0", - "title" => "Data Processing Pipeline", - "steps" => [ - # Extract data from two sources in parallel - %{ - "name" => "extract_users", - "type" => "task", - "command" => "extract", - "args" => %{"source" => "users"} - }, - %{ - "name" => "extract_orders", - "type" => "task", - "command" => "extract", - "args" => %{"source" => "orders"} - }, - # Join them - %{ - "name" => "join", - "type" => "task", - "command" => "merge", - "dependencies" => ["extract_users", "extract_orders"] - }, - # Load to warehouse - %{ - "name" => "load", - "type" => "task", - "command" => "load", - "dependencies" => ["join"] - } - ] - } - end - - @impl true - def execute_command(_run_id, "extract", %{"source" => source}, _ctx) do - # Simulate data extraction - {:ok, %{"items" => 100, "source" => source}} - end - - @impl true - def execute_command(_run_id, "merge", input, _ctx) do - # Merge results from previous steps - {:ok, %{"merged_count" => 200}} - end - - @impl true - def execute_command(_run_id, "load", input, _ctx) do - # Load to warehouse - {:ok, %{"loaded" => true}} - end - - @impl true - def on_complete(_run_id, output, _ctx) do - IO.puts("Pipeline completed!") - end - - @impl true - def on_failure(_run_id, error, _ctx) do - IO.inspect(error, label: "Pipeline failed") - end -end -``` - -## Map Steps (Parallel Iteration) - -Execute the same task across multiple items: - -```elixir -defmodule MyApp.Workflows.ProcessItems do - @behaviour Singularity.Workflow.Executor.Workflow - - @impl true - def definition do - %{ - "version" => "1.0", - "title" => "Process Multiple Items", - "steps" => [ - # Map step: create a task for each item - %{ - "name" => "process_items", - "type" => "map", - "command" => "process", - "over" => "[1, 2, 3, 4, 5]" - }, - # Wait for all to complete - %{ - "name" => "aggregate", - "type" => "task", - "command" => "aggregate", - "dependencies" => ["process_items"] - } - ] - } - end - - @impl true - def execute_command(_run_id, "process", %{"item" => item}, _ctx) do - {:ok, %{"processed" => item, "result" => item * 2}} - end - - @impl true - def execute_command(_run_id, "aggregate", input, _ctx) do - {:ok, %{"aggregated" => true}} - end - - @impl true - def on_complete(_run_id, _output, _ctx), do: :ok - def on_failure(_run_id, _error, _ctx), do: :ok -end -``` - -## Goal-Driven Workflows with HTDAG - -Want workflows that understand **goals** instead of explicit tasks? Use HTDAG (Hierarchical Task DAG) for automatic workflow decomposition. - -### Define a Goal Decomposer - -```elixir -defmodule MyApp.SimpleDecomposer do - def decompose(goal) do - # Break goal into steps (could use rules, LLM, or both) - tasks = [ - %{id: "analyze", description: "Analyze: #{goal}", depends_on: []}, - %{id: "plan", description: "Plan solution", depends_on: ["analyze"]}, - %{id: "execute", description: "Execute plan", depends_on: ["plan"]}, - %{id: "verify", description: "Verify result", depends_on: ["execute"]} - ] - {:ok, tasks} - end -end -``` - -### Define Step Functions - -```elixir -step_functions = %{ - "analyze" => fn input -> - analysis = "Analysis of #{input.goal} complete" - {:ok, %{analysis: analysis}} - end, - "plan" => fn input -> - plan = "Plan based on: #{input.analysis}" - {:ok, %{plan: plan}} - end, - "execute" => fn input -> - result = "Executed: #{input.plan}" - {:ok, %{result: result}} - end, - "verify" => fn input -> - {:ok, %{verified: true, final_result: input.result}} - end -} -``` - -### Execute Goal-Driven Workflow - -```elixir -{:ok, result} = Singularity.Workflow.WorkflowComposer.compose_from_goal( - "Build a user authentication system", - &MyApp.SimpleDecomposer.decompose/1, - step_functions, - MyApp.Repo, - optimization_level: :basic # Start safe, can increase later -) - -IO.inspect(result) -# => %{verified: true, final_result: "Executed: Plan..."} -``` - -### Optimization Levels - -```elixir -# :basic - Safe, conservative optimizations -{:ok, result} = Singularity.Workflow.WorkflowComposer.compose_from_goal( - goal, - &decomposer/1, - steps, - repo, - optimization_level: :basic -) - -# :advanced - Intelligent optimizations based on historical data -{:ok, result} = Singularity.Workflow.WorkflowComposer.compose_from_goal( - goal, - &decomposer/1, - steps, - repo, - optimization_level: :advanced, - monitoring: true -) - -# :aggressive - Maximum optimization (requires 100+ executions) -{:ok, result} = Singularity.Workflow.WorkflowComposer.compose_from_goal( - goal, - &decomposer/1, - steps, - repo, - optimization_level: :aggressive -) -``` - -### Multi-Workflow Composition - -Execute multiple goals in parallel: - -```elixir -goals = [ - "Build authentication service", - "Build payment service", - "Build notification service" -] - -{:ok, results} = Singularity.Workflow.WorkflowComposer.compose_multiple_workflows( - goals, - &MyApp.ServiceDecomposer.decompose/1, - %{ - "auth" => auth_step_functions, - "payment" => payment_step_functions, - "notification" => notification_step_functions - }, - MyApp.Repo, - parallel: true # Execute all three in parallel -) -``` - -For the complete guide, see [HTDAG_ORCHESTRATOR_GUIDE.md](docs/HTDAG_ORCHESTRATOR_GUIDE.md). - -## Configuration - -Singularity.Workflow respects these environment variables: - -```bash -# PostgreSQL connection -DATABASE_URL=postgres://user:pass@localhost:5432/my_app - -# PGMQ queue name (default: singularity_workflow_queue) -SINGULARITY_WORKFLOW_QUEUE_NAME=my_queue - -# Visibility timeout for in-flight tasks (default: 300s = 5 min) -SINGULARITY_WORKFLOW_VT=300 - -# Max concurrent task executions (default: 10) -SINGULARITY_WORKFLOW_MAX_WORKERS=10 -``` - -## Troubleshooting - -### "pgmq extension not found" - -Install the extension: - -```bash -pgxn install pgmq -createdb my_app -psql my_app -c "CREATE EXTENSION IF NOT EXISTS pgmq" -``` - -### "Queue not found" - -Ensure migrations have run: - -```bash -mix ecto.status # Check pending migrations -mix ecto.migrate # Run all migrations -``` - -### Tasks hanging or not executing - -Check queue health: - -```elixir -alias Singularity.Workflow.Executor - -# See how many tasks are pending -{:ok, count} = Executor.pending_task_count() -IO.inspect(count) - -# Manually trigger task processing -Executor.execute_pending_tasks() -``` - -### Type errors in custom workflows - -Singularity.Workflow uses Dialyzer for type checking. Run: - -```bash -mix dialyzer -``` - -## Next Steps - -- Read [ARCHITECTURE.md](docs/ARCHITECTURE.md) for internal design details -- Check [DYNAMIC_WORKFLOWS_GUIDE.md](docs/DYNAMIC_WORKFLOWS_GUIDE.md) for advanced patterns -- See [API_REFERENCE.md](docs/API_REFERENCE.md) for complete API documentation -- Review [SECURITY.md](SECURITY.md) for security policy and best practices - -## Contributing - -Found a bug? Have a feature request? See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. - -## License - -MIT - See [LICENSE.md](LICENSE.md) for details. diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index 9bdd60c..0000000 --- a/RELEASING.md +++ /dev/null @@ -1,198 +0,0 @@ -# Release Process for Singularity.Workflow - -This document describes how to publish a new release of Singularity.Workflow. - -## Two Release Options - -### Option 1: GitHub Release Only (No Hex.pm API Key Required) - -For creating GitHub releases without publishing to Hex.pm: - -```bash -./scripts/release.sh 0.1.0 github -``` - -This uses `.github/workflows/release-github-only.yml` and: -- Runs all CI tests automatically -- Creates GitHub release with changelog -- No Hex.pm credentials needed -- You can publish to Hex.pm manually later - -### Option 2: Full Release (GitHub + Hex.pm) - -For publishing to both GitHub and Hex.pm: - -```bash -./scripts/release.sh 0.1.0 hex -``` - -This uses `.github/workflows/publish.yml` and requires: -- HEX_API_KEY secret configured -- Production environment for manual approval -- Publishes to Hex.pm automatically - -## Prerequisites - -### For GitHub-Only Releases - -No setup required - just push the tag. - -### For Hex.pm Releases - -1. **HEX_API_KEY Secret**: Add your Hex.pm API key to GitHub repository secrets -2. **Production Environment**: Configure manual approval for releases - - Go to repository Settings → Environments - - Create an environment named `production` - - Add required reviewers who can approve releases - -3. **All Changes Committed**: Ensure all changes are committed and pushed - -## Quick Release - -### GitHub Only (Default) - -```bash -./scripts/release.sh 0.1.0 -``` - -or explicitly: - -```bash -./scripts/release.sh 0.1.0 github -``` - -### GitHub + Hex.pm - -```bash -./scripts/release.sh 0.1.0 hex -``` -- Create a git tag `v0.1.0` -- Push the tag to GitHub -- Trigger the automated publish workflow - -## Manual Release Steps - -If you prefer to do it manually: - -### 1. Create and Push Tag - -```bash -# Create annotated tag -git tag -a v0.1.0 -m "Release v0.1.0" - -# Push tag to GitHub -git push origin v0.1.0 -``` - -### 2. GitHub Actions Workflow - -The tag push automatically triggers `.github/workflows/publish.yml` which: - -1. **CI Tests** (automatic): - - Runs full test suite - - Checks code formatting - - Runs Credo quality checks - - Runs security audit with Sobelow - -2. **Manual Approval** (requires human): - - Workflow pauses for manual approval - - Configured reviewers must approve in the GitHub UI - - Go to Actions → Publish to Hex.pm → Review deployments - -3. **Publish** (automatic after approval): - - Publishes package to Hex.pm - - Creates GitHub release with changelog - - Tags release with version - -### 3. Verify Publication - -After the workflow completes: - -- Check Hex.pm: https://hex.pm/packages/singularity_workflow -- Check GitHub releases: https://github.com/Singularity-ng/singularity-workflows/releases - -## Version Preparation Checklist - -Before creating a release tag, ensure: - -- [ ] Version updated in `mix.exs` -- [ ] `CHANGELOG.md` updated with release notes -- [ ] `README.md` references correct version -- [ ] All tests passing (`mix test`) -- [ ] Code quality checks passing (`mix quality`) -- [ ] Documentation reviewed and updated -- [ ] Security audit passing (`mix sobelow`) - -## Troubleshooting - -### Tag Already Exists - -If you need to re-release: - -```bash -# Delete local tag -git tag -d v0.1.0 - -# Delete remote tag -git push origin :refs/tags/v0.1.0 - -# Recreate tag -git tag -a v0.1.0 -m "Release v0.1.0" -git push origin v0.1.0 -``` - -### Workflow Fails - -Check the GitHub Actions logs: -- Go to repository → Actions tab -- Find the failed workflow run -- Review error messages in each job - -Common issues: -- Missing `HEX_API_KEY` secret -- Test failures (check PostgreSQL service) -- Missing approval environment configuration - -### Hex.pm Authentication Issues - -Ensure your `HEX_API_KEY` secret: -- Is valid and not expired -- Has write permissions for the package -- Is properly configured in repository secrets - -## Post-Release Tasks - -After successful release: - -1. Announce the release (optional) -2. Update any dependent projects -3. Close related GitHub issues -4. Update project board/roadmap - -## Local Testing Before Release - -Test the package locally before releasing: - -```bash -# Clean build -mix deps.clean --all -mix clean - -# Reinstall and test -mix deps.get -mix test - -# Check package content -mix hex.build -tar tzf singularity_workflow-0.1.0.tar | head -20 -``` - -## Emergency Rollback - -If you need to yank a release from Hex.pm: - -```bash -mix hex.retire singularity_workflow 0.1.0 --reason security -``` - -Note: Yanking doesn't delete the release, it just marks it as retired. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 58fe2b9..e455075 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -853,7 +853,7 @@ See `test/` directory for examples. 5. **Monitoring**: Check pgmq queue depth, step failure rate 6. **Backup**: PostgreSQL WAL archiving for durability -See [GETTING_STARTED.md](GETTING_STARTED.md) for deployment steps. +See the main [README](../README.md) for installation and setup instructions, or [DEPLOYMENT_GUIDE](DEPLOYMENT_GUIDE.md) for production deployment steps. ## Implementation Details diff --git a/docs/README.md b/docs/README.md index 7b56ab6..17a7cbd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,8 +8,7 @@ Complete documentation for the Singularity.Workflow library. ### Getting Started -- **[README](../README.md)** - Project overview, features, and quick start -- **[GETTING_STARTED](../GETTING_STARTED.md)** - Installation and first workflow +- **[README](../README.md)** - Project overview, features, quick start, and installation guide - **[CHANGELOG](../CHANGELOG.md)** - Version history and release notes ### Core Documentation @@ -71,7 +70,7 @@ Complete documentation for the Singularity.Workflow library. ### I want to... **Start using the library:** -→ [README](../README.md) → [GETTING_STARTED](../GETTING_STARTED.md) +→ [README](../README.md) (includes installation and quick start) **Understand the API:** → [API_REFERENCE](API_REFERENCE.md) diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index cc66782..3d24404 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -373,6 +373,6 @@ mix test --trace --verbose ## Documentation References -- **[GETTING_STARTED.md](GETTING_STARTED.md)** - Setup instructions +- **[README](../README.md)** - Installation and setup instructions - **[ARCHITECTURE.md](ARCHITECTURE.md)** - Technical design - **[DEPLOYMENT_GUIDE.md](DEPLOYMENT_GUIDE.md)** - Production deployment