From 5a6a3e7abbcbc28b164cfa3944cd98eabb3cc616 Mon Sep 17 00:00:00 2001 From: Dimitar Atanasov Date: Thu, 21 May 2026 14:21:26 +0300 Subject: [PATCH 1/3] fix(ci): rename $args to $dockerArgs in common.ps1 Invoke-Worker PowerShell's $args is an automatic variable that PowerShell auto-populates with positional arguments passed to a function or script. Assigning to it inside Invoke-Worker triggered 9 PSAvoidAssignmentToAutomaticVariable warnings from PSScriptAnalyzer, which the lint workflow treats as errors and fails the CI run on. Rename the local docker-args accumulator to $dockerArgs throughout Invoke-Worker. Behaviour is unchanged (the variable was never read back through the automatic-variable mechanism); only the symbol name changes. This matches the convention already used in scripts/powershell/08-run-worker.ps1 where the same accumulator is called $dockerArgs. --- scripts/powershell/common.ps1 | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/powershell/common.ps1 b/scripts/powershell/common.ps1 index edd42b7..92f0c62 100644 --- a/scripts/powershell/common.ps1 +++ b/scripts/powershell/common.ps1 @@ -53,9 +53,9 @@ function Invoke-Worker { $keystoreFile = Get-KeystoreFileName - $args = @("run", "--rm") - $args += $ExtraRunFlags - $args += @( + $dockerArgs = @("run", "--rm") + $dockerArgs += $ExtraRunFlags + $dockerArgs += @( "-v", "$($env:KEYS_DIR):/data", "-e", "WORKER_KEYSTORE_PATH=/data/eth-keystore/$keystoreFile", "-e", "WORKER_KEYSTORE_PASSWORD=$env:WORKER_PASSWORD", @@ -67,22 +67,22 @@ function Invoke-Worker { "-e", "SUPPORTED_MODELS=$env:SUPPORTED_MODELS" ) if ($env:JOB_REGISTRY_ADDRESS) { - $args += @("-e", "JOB_REGISTRY_ADDRESS=$env:JOB_REGISTRY_ADDRESS") + $dockerArgs += @("-e", "JOB_REGISTRY_ADDRESS=$env:JOB_REGISTRY_ADDRESS") } - foreach ($pair in $ExtraEnv) { $args += @("-e", $pair) } + foreach ($pair in $ExtraEnv) { $dockerArgs += @("-e", $pair) } - if (-not $NoEntrypoint) { $args += @("--entrypoint", "/bin/lightchain-worker") } + if (-not $NoEntrypoint) { $dockerArgs += @("--entrypoint", "/bin/lightchain-worker") } - $args += $env:IMAGE - if ($Subcommand) { $args += $Subcommand } - $args += $ExtraArgs + $dockerArgs += $env:IMAGE + if ($Subcommand) { $dockerArgs += $Subcommand } + $dockerArgs += $ExtraArgs - $redactedArgs = $args | ForEach-Object { + $redactedArgs = $dockerArgs | ForEach-Object { $_ -replace '(WORKER_KEYSTORE_PASSWORD=)\S+', '$1***' ` -replace '(--private-key\s+)0x[a-fA-F0-9]+', '$1***' } Write-Host "+ docker $($redactedArgs -join ' ')" -ForegroundColor DarkGray - & docker @args + & docker @dockerArgs } function Set-ResolvedVar { From 4aab86e16e52080c7f820be82342b0299cc9bbf3 Mon Sep 17 00:00:00 2001 From: Dimitar Atanasov Date: Fri, 22 May 2026 10:04:22 +0300 Subject: [PATCH 2/3] fix(ci): extract markdownlint config to a real file so the workflow runs The previous workflow inlined the JSON config under markdownlint-cli2-action's 'config:' input, but that input expects a file *path*. The action then tried to read a file whose entire name was the JSON blob, and every run failed at config-load time before any lint ran: Error: Unable to use configuration file '/home/runner/.../{ "default": true, ... Move the same config into .markdownlint-cli2.jsonc at the repo root (an auto-discovered location for markdownlint-cli2), and drop the broken 'with:' block from the workflow step. The action now picks the config up on its own. --- .github/workflows/lint.yml | 11 +---------- .markdownlint-cli2.jsonc | 11 +++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 .markdownlint-cli2.jsonc diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 14a510d..06daae1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -61,13 +61,4 @@ jobs: - uses: actions/checkout@v4 - uses: DavidAnson/markdownlint-cli2-action@v16 - with: - globs: "**/*.md" - config: | - { - "default": true, - "MD013": false, - "MD024": { "siblings_only": true }, - "MD033": false, - "MD041": false - } + # globs + config are auto-discovered from .markdownlint-cli2.jsonc in the repo root. diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000..9565bc6 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,11 @@ +{ + "config": { + "default": true, + "MD013": false, + "MD024": { "siblings_only": true }, + "MD033": false, + "MD041": false + }, + "globs": ["**/*.md"], + "ignores": ["node_modules"] +} From 2e14dec4b06dce2898475d1260fa263c5d920018 Mon Sep 17 00:00:00 2001 From: Dimitar Atanasov Date: Fri, 22 May 2026 10:04:38 +0300 Subject: [PATCH 3/3] style(docs): fix 137 pre-existing markdownlint violations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The markdownlint CI workflow has been broken since the repo was created (the config-loading bug fixed in the previous commit). With the workflow now functional, it surfaces 137 real lint violations across 12 doc files that have accumulated. This commit fixes all of them. Breakdown by rule: * MD060 (96) — table-column-style: pipes did not align with header for 'aligned' style. Reformatted every table in README.md, all docs/*.md, and faq.md so every pipe aligns vertically. Properly accounts for the wide-character ✅ (East-Asian-Width = W) used in the README's status table. * MD040 (23) — fenced-code-language: bare ``` blocks now have an explicit language tag. Most became 'text' (log output, error messages); installation-linux.md:125 is a systemd unit so it became 'ini'. * MD031 (6) — blanks-around-fences: code fences need blank lines around them. Auto-fixed. * MD022 (4) — blanks-around-headings: ditto for headings. Auto-fixed. * MD032 (4) — blanks-around-lists: ditto for lists. Auto-fixed. * MD034 (2) — no-bare-urls: bare URLs in faq.md wrapped with <>. Auto-fixed. * MD051 (1) — link-fragments: troubleshooting.md had a broken anchor pointing at #9-windows-host.docker.internal-ipv6-first; GitHub strips the periods, so the correct anchor is #9-windows-hostdockerinternal- ipv6-first. * MD028 (1) — no-blanks-blockquote: two adjacent blockquotes in security.md joined with a '>' line in between (preserves visual paragraph break). * MD027 (1) — no-multiple-space-blockquote: README's lead blockquote had a double space after '>'. Auto-fixed. Net result: markdownlint passes with 0 errors. No rules disabled. --- CHANGELOG.md | 4 ++ CONTRIBUTING.md | 2 +- README.md | 106 +++++++++++++++++------------------ docs/architecture.md | 78 +++++++++++++------------- docs/faq.md | 16 +++--- docs/installation-linux.md | 4 +- docs/installation-macos.md | 28 ++++----- docs/installation-windows.md | 10 ++-- docs/onboarding.md | 16 +++--- docs/operations.md | 2 +- docs/security.md | 18 ++++-- docs/troubleshooting.md | 64 ++++++++++----------- 12 files changed, 179 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e253f7a..c26ddf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] ### Added + - Bash scripts (`scripts/bash/`) — full parity with the PowerShell flow, for Linux and macOS users. - `docs/architecture.md` — sequence diagram of the 8-stage job lifecycle, component glossary, and the metrics reference. - `docs/installation-{windows,linux,macos}.md` — per-OS install guides with verified commands. @@ -20,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Issue and PR templates. ### Fixed + - `SUPPORTED_MODELS` default switched from `llama3-8b:latest` back to `llama3-8b`. The `:latest` suffix passed the local startup health check but caused every routed job to fail at stage 5 because `keccak256("llama3-8b:latest")` doesn't match the on-chain registered model hash (`keccak256("llama3-8b") = 0xf4a414fa...`). See [troubleshooting issue #1](./docs/troubleshooting.md#1-model-hash-mismatch-stage-5-failure). - `OLLAMA_URL` on Windows now defaults to the explicit IPv4 `http://192.168.65.254:11434` rather than `host.docker.internal`. On Docker Desktop for Windows, the latter resolves IPv6-first, which can stall Go's HTTP client. See [troubleshooting issue #9](./docs/troubleshooting.md#9-windows-host.docker.internal-ipv6-first). - `08-run-worker.ps1` now accepts `-NoTail` to skip the blocking `docker logs -f` call after starting the container (useful in automation). @@ -27,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - `deregister.ps1` now accepts `-Force` for the same reason. ### Changed + - Project layout: PowerShell scripts moved from `scripts/*.ps1` to `scripts/powershell/`, with `scripts/bash/` added alongside. - `WORKER_PASSWORD` is now generated as a 32-character cryptographically random string on first run of `00-generate-key.ps1`, instead of relying on the user to choose one. - `secrets.{ps1,env}` is created with restrictive ACLs (Windows: current user only via `Set-Acl`; Linux/macOS: `chmod 600`) automatically. @@ -37,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), Initial PowerShell port of the [official Lightchain worker onboarding](https://workers-testnet.lightchain.ai/run-node). ### Added + - `scripts/*.ps1` covering phases 00-08 (key gen, contract resolve, ollama prep, image pull, key import, ECDH gen, fund, register, run). - `scripts/status.ps1`, `scripts/stop.ps1`, `scripts/deregister.ps1`, `scripts/sweep-rewards.ps1`, `scripts/tail-jobs.ps1` for day-2 ops. - `scripts/common.ps1` with shared `Invoke-Worker` helper. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2d6e51..a814eb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,7 +117,7 @@ If your change is invasive (new env var, new script, new dependency), describe y Conventional Commits style preferred but not enforced: -``` +```text fix: handle host.docker.internal IPv6-first on Windows docs: add stage 5 hash-mismatch entry to troubleshooting feat: add sweep-rewards --dry-run diff --git a/README.md b/README.md index ab6d628..75808f0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # lightchain-worker-toolkit -> [Lightchain](https://lightchain.ai) worker on your own hardware — Windows, macOS, or Linux — with every gotcha we hit in production already fixed. +> [Lightchain](https://lightchain.ai) worker on your own hardware — Windows, macOS, or Linux — with every gotcha we hit in production already fixed. [![lint](https://img.shields.io/github/actions/workflow/status/lightchain/lightchain-worker-toolkit/lint.yml?branch=main&label=lint)](./.github/workflows/lint.yml) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) @@ -34,11 +34,11 @@ This repo is the **PowerShell + Bash automation** I wish existed when I onboarde ## What you get -| Layer | What's in this repo | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------- | -| **Scripts** | 16 PowerShell scripts + 16 Bash scripts that automate every phase of the official onboarding (key gen → register → run) and the day-2 ops (status, sweep, deregister, tail). | -| **Docs** | 9 guides covering architecture, per-OS installation, deep dives into each phase, day-2 operations, troubleshooting, security, and FAQ. | -| **Examples** | systemd unit for unattended Linux operation, docker-compose alternative, full env-var reference. | +| Layer | What's in this repo | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Scripts** | 16 PowerShell scripts + 16 Bash scripts that automate every phase of the official onboarding (key gen → register → run) and the day-2 ops (status, sweep, deregister, tail). | +| **Docs** | 9 guides covering architecture, per-OS installation, deep dives into each phase, day-2 operations, troubleshooting, security, and FAQ. | +| **Examples** | systemd unit for unattended Linux operation, docker-compose alternative, full env-var reference. | | **Repo hygiene** | MIT license, CONTRIBUTING guide, SECURITY policy, CHANGELOG, .gitignore that actually protects your keys, .gitattributes for cross-OS line endings, .editorconfig, GitHub Actions for PowerShell + Bash + Markdown linting, issue/PR templates. | Everything in `scripts/` is **idempotent**: each phase script can be safely re-run, and refuses to do destructive things (overwrite keystore, send LCAI to wrong address) without explicit confirmation. @@ -54,16 +54,16 @@ Everything in `scripts/` is **idempotent**: each phase script can be safely re-r ## Status -| Capability | Status | -| ------------------------------- | ----------------------------------------------------- | -| Mainnet onboarding (Windows) | ✅ Verified end-to-end on RTX 5060 Ti + Windows 11. | -| Mainnet onboarding (Linux) | ✅ Verified on Ubuntu 24.04 + RTX 4070. | -| Mainnet onboarding (macOS) | ✅ Verified on M2 Pro 16GB. | -| Testnet onboarding | ✅ Same scripts, flip `NETWORK=testnet`. | +| Capability | Status | +| ------------------------------- | ----------------------------------------------------------------------------------- | +| Mainnet onboarding (Windows) | ✅ Verified end-to-end on RTX 5060 Ti + Windows 11. | +| Mainnet onboarding (Linux) | ✅ Verified on Ubuntu 24.04 + RTX 4070. | +| Mainnet onboarding (macOS) | ✅ Verified on M2 Pro 16GB. | +| Testnet onboarding | ✅ Same scripts, flip `NETWORK=testnet`. | | Multi-worker on one host | ✅ Documented in [operations.md](./docs/operations.md#multi-worker-on-one-machine). | -| Job processing | ✅ Confirmed (stage 1 → 8 → `job completed`). | -| Sweep / deregister | ✅ Working. | -| Auto-restart on host reboot | ✅ Container has `--restart always`; systemd example included. | +| Job processing | ✅ Confirmed (stage 1 → 8 → `job completed`). | +| Sweep / deregister | ✅ Working. | +| Auto-restart on host reboot | ✅ Container has `--restart always`; systemd example included. | ## Quick start (TL;DR) @@ -131,7 +131,7 @@ export FUNDER_PRIVKEY=0xYOUR_FUNDER_KEY ## Repository layout -``` +```text . ├── README.md # ← you are here ├── LICENSE # MIT @@ -221,37 +221,37 @@ flowchart LR P8 -.->|long-running| D[Docker container] ``` -| # | Phase | What changes | Reversible? | Time | -| -- | ---------------------- | ----------------------------------------------------- | ------------------------------------------------------------------- | --------- | -| 00 | Generate worker key | New secp256k1 keypair, stored in `secrets.ps1` | Yes (delete the file, generate a fresh one) | ~1 sec | -| 01 | Resolve addresses | Reads 2 on-chain proxy addresses, persists locally | Yes (just re-run) | ~1 sec | -| 02 | Prepare Ollama | Pulls `llama3:8b` if missing, creates `llama3-8b` alias| Yes (`ollama rm llama3-8b`) | 30s-5min | -| 03 | Pull image | Downloads ~95 MB worker Docker image | Yes (`docker rmi`) | 30s-2min | -| 04 | Import key | Writes encrypted keystore file to disk | Yes (delete file, re-run) | ~5 sec | -| 05 | Generate ECDH | Writes `worker-encryption.key` to disk | Yes BUT requires re-register if already registered | ~5 sec | -| 06 | Fund worker | **On-chain tx**: sends 50,005 LCAI funder → worker | Yes (sweep back) | ~10 sec | -| 07 | Register | **On-chain tx**: stakes 50,000 LCAI | Yes (`deregister` returns the stake minus any slashing) | ~10 sec | -| 08 | Run sidecar | Starts long-running container | Yes (`stop.ps1` removes it; registration + stake stay on-chain) | ~3 sec | +| # | Phase | What changes | Reversible? | Time | +| --- | ---------------------- | ------------------------------------------------------- | ------------------------------------------------------------------- | --------- | +| 00 | Generate worker key | New secp256k1 keypair, stored in `secrets.ps1` | Yes (delete the file, generate a fresh one) | ~1 sec | +| 01 | Resolve addresses | Reads 2 on-chain proxy addresses, persists locally | Yes (just re-run) | ~1 sec | +| 02 | Prepare Ollama | Pulls `llama3:8b` if missing, creates `llama3-8b` alias | Yes (`ollama rm llama3-8b`) | 30s-5min | +| 03 | Pull image | Downloads ~95 MB worker Docker image | Yes (`docker rmi`) | 30s-2min | +| 04 | Import key | Writes encrypted keystore file to disk | Yes (delete file, re-run) | ~5 sec | +| 05 | Generate ECDH | Writes `worker-encryption.key` to disk | Yes BUT requires re-register if already registered | ~5 sec | +| 06 | Fund worker | **On-chain tx**: sends 50,005 LCAI funder → worker | Yes (sweep back) | ~10 sec | +| 07 | Register | **On-chain tx**: stakes 50,000 LCAI | Yes (`deregister` returns the stake minus any slashing) | ~10 sec | +| 08 | Run sidecar | Starts long-running container | Yes (`stop.ps1` removes it; registration + stake stay on-chain) | ~3 sec | ## Costs and economics ### Upfront, one-time (mainnet) -| Item | Amount | -| ------------------------------------ | -------------------- | -| Stake (locked in WorkerRegistry) | **50,000 LCAI** | -| Funding tx gas | < 0.01 LCAI | -| Registration tx gas | < 0.05 LCAI | -| Gas buffer in worker wallet | ~5 LCAI (recommended)| -| **Total to fund** | **~50,005 LCAI** | +| Item | Amount | +| ------------------------------------ | --------------------- | +| Stake (locked in WorkerRegistry) | **50,000 LCAI** | +| Funding tx gas | < 0.01 LCAI | +| Registration tx gas | < 0.05 LCAI | +| Gas buffer in worker wallet | ~5 LCAI (recommended) | +| **Total to fund** | **~50,005 LCAI** | ### Per-job -| Item | Amount | -| ------------------------------------ | ------------------- | -| Gas for `ackJob` | < 0.0005 LCAI | -| Gas for `completeJob` | < 0.0005 LCAI | -| Reward | varies by `AIConfig`| +| Item | Amount | +| ------------------------------------ | -------------------- | +| Gas for `ackJob` | < 0.0005 LCAI | +| Gas for `completeJob` | < 0.0005 LCAI | +| Reward | varies by `AIConfig` | ### Recoverable when you exit @@ -282,18 +282,18 @@ Cards with < 8 GB VRAM will OOM loading the model. CPU-only inference is too slo ## Network reference -| Resource | Mainnet | Testnet (verify on the [run-node site](https://workers-testnet.lightchain.ai/run-node))| -| --------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| Chain ID | `9200` | `8200` | -| Chain RPC | `https://rpc.mainnet.lightchain.ai` | `https://rpc.testnet.lightchain.ai` | -| Beacon API | `https://beacon.mainnet.lightchain.ai` | `https://beacon.testnet.lightchain.ai` | -| Worker Gateway | `https://worker-gateway.mainnet.lightchain.ai` | `https://worker-gateway.testnet.lightchain.ai` | -| Dispatcher | `https://dispatcher.mainnet.lightchain.ai` | - | -| Status page | `https://status.mainnet.lightchain.ai` | - | -| Worker image | `us-central1-docker.pkg.dev/lightchain/lightchain-mainnet-public-docker/worker:latest` | `us-central1-docker.pkg.dev/lightchain/lightchain-testnet-public-docker/worker:latest` | -| `WorkerRegistry` | `0x0000000000000000000000000000000000001002` (genesis predeploy, same on both) | same | -| `AIConfig` proxy | `0x24D11533C354092ed6E18b964257819cE78Ce77D` | resolve via `cast call $WORKER_REGISTRY_ADDRESS "aiConfig()(address)"` | -| `JobRegistry` proxy | `0xfB15F90298e4CcD7106E76fFB5e520315cC42B0b` | resolve via `cast call $WORKER_REGISTRY_ADDRESS "jobRegistry()(address)"` | +| Resource | Mainnet | Testnet (verify on the [run-node site](https://workers-testnet.lightchain.ai/run-node)) | +| --------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| Chain ID | `9200` | `8200` | +| Chain RPC | `https://rpc.mainnet.lightchain.ai` | `https://rpc.testnet.lightchain.ai` | +| Beacon API | `https://beacon.mainnet.lightchain.ai` | `https://beacon.testnet.lightchain.ai` | +| Worker Gateway | `https://worker-gateway.mainnet.lightchain.ai` | `https://worker-gateway.testnet.lightchain.ai` | +| Dispatcher | `https://dispatcher.mainnet.lightchain.ai` | - | +| Status page | `https://status.mainnet.lightchain.ai` | - | +| Worker image | `us-central1-docker.pkg.dev/lightchain/lightchain-mainnet-public-docker/worker:latest` | `us-central1-docker.pkg.dev/lightchain/lightchain-testnet-public-docker/worker:latest` | +| `WorkerRegistry` | `0x0000000000000000000000000000000000001002` (genesis predeploy, same on both) | same | +| `AIConfig` proxy | `0x24D11533C354092ed6E18b964257819cE78Ce77D` | resolve via `cast call $WORKER_REGISTRY_ADDRESS "aiConfig()(address)"` | +| `JobRegistry` proxy | `0xfB15F90298e4CcD7106E76fFB5e520315cC42B0b` | resolve via `cast call $WORKER_REGISTRY_ADDRESS "jobRegistry()(address)"` | `AIConfig` and `JobRegistry` are upgradeable proxies; the toolkit resolves them at runtime via Phase 01, so they auto-track any protocol upgrade. @@ -319,7 +319,7 @@ These are real failure modes we hit while running a mainnet worker. The toolkit The worker computes `keccak256(SUPPORTED_MODELS)` locally and uses that hash to look up incoming jobs. The gateway sends jobs keyed by `keccak256("llama3-8b") = 0xf4a414fa...`. If your `SUPPORTED_MODELS` is anything else (`"llama3-8b:latest"`, `"llama3:8b"`, `"Llama3-8b"`), the hash table miss causes **every** routed job to fail at stage 5 with: -``` +```text ERROR job failed - stage 5 (resolve model): no local Ollama model configured for queued model ID "0xf4a414fa..." ``` @@ -331,7 +331,7 @@ On Docker Desktop for Windows, `host.docker.internal` often resolves to an IPv6 ### 3. The benign `WARN` at startup -``` +```text WARN ollama model verification failed (non-fatal) - missing models on Ollama server: [llama3-8b] ``` diff --git a/docs/architecture.md b/docs/architecture.md index 4673c4a..a985959 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -38,39 +38,39 @@ sequenceDiagram ## Component glossary -| Component | What it is | Where it runs | -| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | -| **Lightchain L1** | EVM-compatible PoS chain. Mainnet chain ID `9200`, testnet `8200`. | Lightchain Foundation | -| **WorkerRegistry** (`0x...001002`) | Genesis predeploy. Holds stake, ECDH pubkey, and the list of supported model hashes per worker. | On-chain | -| **AIConfig** (proxy) | Holds protocol params (min stake, slashing rates, supported model whitelist). Read via `WorkerRegistry.aiConfig()`. | On-chain, upgradeable | -| **JobRegistry** (proxy) | Records each job's lifecycle (created → acked → completed). Read via `WorkerRegistry.jobRegistry()`. | On-chain, upgradeable | -| **Beacon API** | EIP-4844 blob storage. Prompts and responses are stored as blobs to keep on-chain calldata cheap. | Lightchain-hosted | -| **Worker Gateway** | WebSocket-based scheduler that pushes jobs to live workers. Tracks worker liveness (some combination of WS pings, on-chain registration, and Redis heartbeats). | Lightchain-hosted | -| **Dispatcher** | HTTP/REST front door for inference requests. Creates jobs in JobRegistry on behalf of requesters. | Lightchain-hosted | -| **Worker sidecar** (`/bin/worker` in the image) | Long-running Go daemon. Maintains WS to gateway, processes assigned jobs through 8 stages, runs a reconciler for stale jobs, exposes Prometheus metrics on `127.0.0.1:9101`. | **Your machine**, Docker container | -| **Worker CLI** (`/bin/lightchain-worker` in the image) | One-shot subcommands: `import-key`, `keygen`, `register`, `add-models`, `deregister`, `status`, `balance`, `withdraw`, `release`. | **Your machine**, Docker container | -| **Ollama** | Local LLM runtime. Worker calls `POST /api/generate` over HTTP on the host's `:11434`. | **Your machine**, native (not Docker) | +| Component | What it is | Where it runs | +| ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | +| **Lightchain L1** | EVM-compatible PoS chain. Mainnet chain ID `9200`, testnet `8200`. | Lightchain Foundation | +| **WorkerRegistry** (`0x...001002`) | Genesis predeploy. Holds stake, ECDH pubkey, and the list of supported model hashes per worker. | On-chain | +| **AIConfig** (proxy) | Holds protocol params (min stake, slashing rates, supported model whitelist). Read via `WorkerRegistry.aiConfig()`. | On-chain, upgradeable | +| **JobRegistry** (proxy) | Records each job's lifecycle (created → acked → completed). Read via `WorkerRegistry.jobRegistry()`. | On-chain, upgradeable | +| **Beacon API** | EIP-4844 blob storage. Prompts and responses are stored as blobs to keep on-chain calldata cheap. | Lightchain-hosted | +| **Worker Gateway** | WebSocket-based scheduler that pushes jobs to live workers. Tracks worker liveness (some combination of WS pings, on-chain registration, and Redis heartbeats). | Lightchain-hosted | +| **Dispatcher** | HTTP/REST front door for inference requests. Creates jobs in JobRegistry on behalf of requesters. | Lightchain-hosted | +| **Worker sidecar** (`/bin/worker` in the image) | Long-running Go daemon. Maintains WS to gateway, processes assigned jobs through 8 stages, runs a reconciler for stale jobs, exposes Prometheus metrics on `127.0.0.1:9101`. | **Your machine**, Docker container | +| **Worker CLI** (`/bin/lightchain-worker` in the image) | One-shot subcommands: `import-key`, `keygen`, `register`, `add-models`, `deregister`, `status`, `balance`, `withdraw`, `release`. | **Your machine**, Docker container | +| **Ollama** | Local LLM runtime. Worker calls `POST /api/generate` over HTTP on the host's `:11434`. | **Your machine**, native (not Docker) | ## The 8 stages of job processing Every job logged by the worker walks through these stages. The stage names below match what you'll see in `docker logs`. -| Stage | Name | What happens | Failure modes | -| ----- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | -| 1 | `ack` | Send `ackJob(jobId)` on-chain. Tells the registry you've started work and locks you to a deadline. | Insufficient gas, RPC down, nonce stuck | -| 2 | `fetch_blob` | Pull the prompt blob from the Beacon API by hash. | Beacon API unreachable, blob expired | -| 3 | `session_key` | Derive the per-session ECDH key from the session registry on-chain. | Session not found, ECDH curve mismatch | -| 4 | `decrypt` | Decrypt the prompt blob with the session key. | Wrong key, malformed ciphertext | -| 5 | `resolve_model`| Look up the on-chain `modelHash` (32-byte `keccak256(name)`) in the worker's local hash→name map (derived from `SUPPORTED_MODELS`). | **Local `SUPPORTED_MODELS` doesn't match on-chain registered name.** This is the #1 gotcha — see [troubleshooting.md](./troubleshooting.md). | -| 6 | `inference` | Call `POST http://OLLAMA_URL/api/generate { model, prompt, stream:false }`. This is where the GPU does actual work — seconds to minutes for llama3-8b. | Ollama down, model missing, OOM, context too long | -| 7 | `publish` | Push the response to Redis (best-effort; on-chain blob is authoritative). | Marked non-fatal — failure increments `worker_redis_publish_failures_total` | -| 8 | `submit_blob` | Submit the response blob via `completeJob(jobId, responseBlobHash)`. | Insufficient gas, deadline expired, on-chain reverted | +| Stage | Name | What happens | Failure modes | +| ----- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `ack` | Send `ackJob(jobId)` on-chain. Tells the registry you've started work and locks you to a deadline. | Insufficient gas, RPC down, nonce stuck | +| 2 | `fetch_blob` | Pull the prompt blob from the Beacon API by hash. | Beacon API unreachable, blob expired | +| 3 | `session_key` | Derive the per-session ECDH key from the session registry on-chain. | Session not found, ECDH curve mismatch | +| 4 | `decrypt` | Decrypt the prompt blob with the session key. | Wrong key, malformed ciphertext | +| 5 | `resolve_model` | Look up the on-chain `modelHash` (32-byte `keccak256(name)`) in the worker's local hash→name map (derived from `SUPPORTED_MODELS`). | **Local `SUPPORTED_MODELS` doesn't match on-chain registered name.** This is the #1 gotcha — see [troubleshooting.md](./troubleshooting.md). | +| 6 | `inference` | Call `POST http://OLLAMA_URL/api/generate { model, prompt, stream:false }`. This is where the GPU does actual work — seconds to minutes for llama3-8b. | Ollama down, model missing, OOM, context too long | +| 7 | `publish` | Push the response to Redis (best-effort; on-chain blob is authoritative). | Marked non-fatal — failure increments `worker_redis_publish_failures_total` | +| 8 | `submit_blob` | Submit the response blob via `completeJob(jobId, responseBlobHash)`. | Insufficient gas, deadline expired, on-chain reverted | Successful completion logs `job completed` with `payoutWei` and credits the reward to the worker wallet. ## What runs where (yours vs Lightchain's) -``` +```text +--------------------- Your machine ---------------------+ | | | Docker container (lightchain-worker) | @@ -111,11 +111,11 @@ Successful completion logs `job completed` with `payoutWei` and credits the rewa A common confusion when first registering: the **stake** and the **rewards** both live in / interact with the same worker wallet address, but they're held differently. -| Position | Where the LCAI sits | Move-able? | -| ----------- | -------------------------------------------------- | --------------------------------------------------------------------- | -| **Stake** | Locked inside `WorkerRegistry` contract, attributed to your worker address | No, until you call `deregister`. Returns to the worker wallet then. | -| **Rewards** | Sit directly in the worker wallet's balance | Yes, immediately. Sweep them to a cold wallet with `sweep-rewards`. | -| **Gas** | Sits in the worker wallet's balance | Yes, but the worker needs ~tiny amounts per job for `ack` and `complete` txs. Don't sweep to zero. | +| Position | Where the LCAI sits | Move-able? | +| ----------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| **Stake** | Locked inside `WorkerRegistry` contract, attributed to your worker address | No, until you call `deregister`. Returns to the worker wallet then. | +| **Rewards** | Sit directly in the worker wallet's balance | Yes, immediately. Sweep them to a cold wallet with `sweep-rewards`. | +| **Gas** | Sits in the worker wallet's balance | Yes, but the worker needs ~tiny amounts per job for `ack` and `complete` txs. Don't sweep to zero. | The toolkit's [`sweep-rewards`](../scripts/powershell/sweep-rewards.ps1) script knows to leave a default 1 LCAI gas buffer behind. @@ -140,17 +140,17 @@ The sidecar exposes Prometheus metrics at `127.0.0.1:9101/metrics` **inside the Notable gauges and counters: -| Metric | What it means | -| --------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| `worker_active_jobs` | In-flight job count. | -| `worker_max_jobs` | Max concurrent jobs (defaults to 2). | +| Metric | What it means | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `worker_active_jobs` | In-flight job count. | +| `worker_max_jobs` | Max concurrent jobs (defaults to 2). | | `worker_ollama_up` | 1 if the periodic Ollama health-check returned 200, 0 otherwise. **Often stays 0 on idle workers — see troubleshooting.** | -| `worker_heartbeat_last_emit_timestamp_seconds` | Unix ts of the last successful heartbeat write to Redis. **Stays 0 if Redis isn't configured.** | -| `worker_release_pending` | Jobs awaiting on-chain release in the local store. | -| `worker_release_released_total` | Cumulative jobs successfully released. | -| `worker_release_reconcile_last_block` | Highest block scanned by the release reconciler. | -| `worker_subpool_inflight{class="blob"\|"legacy"}` | Pending tx broadcasts in the subpool coordinator. | -| `worker_stuck_nonce_tracked` | Nonces stuck in mempool. | -| `worker_redis_publish_failures_total` | Stage 7 PUBLISH failures (non-fatal). | +| `worker_heartbeat_last_emit_timestamp_seconds` | Unix ts of the last successful heartbeat write to Redis. **Stays 0 if Redis isn't configured.** | +| `worker_release_pending` | Jobs awaiting on-chain release in the local store. | +| `worker_release_released_total` | Cumulative jobs successfully released. | +| `worker_release_reconcile_last_block` | Highest block scanned by the release reconciler. | +| `worker_subpool_inflight{class="blob"\|"legacy"}` | Pending tx broadcasts in the subpool coordinator. | +| `worker_stuck_nonce_tracked` | Nonces stuck in mempool. | +| `worker_redis_publish_failures_total` | Stage 7 PUBLISH failures (non-fatal). | See [`docs/troubleshooting.md`](./troubleshooting.md) for what to do when these numbers look wrong. diff --git a/docs/faq.md b/docs/faq.md index eea9c01..411c56b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -4,11 +4,11 @@ ### Is this toolkit official? -No. It's an independent community toolkit. The official documentation is at https://workers-testnet.lightchain.ai/run-node. This toolkit automates and extends that flow, with PowerShell support for Windows users and battle-tested fixes for the gotchas we hit in production. +No. It's an independent community toolkit. The official documentation is at . This toolkit automates and extends that flow, with PowerShell support for Windows users and battle-tested fixes for the gotchas we hit in production. ### Mainnet or testnet — which should I start with? -**Testnet first**, if it's your first time. Testnet LCAI comes free from the faucet (https://lightfaucet.ai/), so you can practice the full onboarding with zero financial risk. Once you're comfortable, switch `$env:NETWORK = "mainnet"` and follow the same flow with real LCAI. +**Testnet first**, if it's your first time. Testnet LCAI comes free from the faucet (), so you can practice the full onboarding with zero financial risk. Once you're comfortable, switch `$env:NETWORK = "mainnet"` and follow the same flow with real LCAI. ### How much can I earn? @@ -157,12 +157,12 @@ Lightchain uses EIP-4844 blobs for prompts and responses (rather than calldata) ### What's the difference between the Worker Gateway and the Dispatcher? -| | Worker Gateway | Dispatcher | -|---|---|---| -| Protocol | WebSocket | HTTPS / REST | -| Talks to | Workers (push jobs) | Requesters (accept inference requests) | -| Purpose | Schedules + load-balances jobs to live workers | Front-door for inference clients; creates JobRegistry entries | -| Worker env var | `WORKER_GATEWAY_URL` | (not used by worker) | +| | Worker Gateway | Dispatcher | +| -------------- | ---------------------------------------------- | ------------------------------------------------------------- | +| Protocol | WebSocket | HTTPS / REST | +| Talks to | Workers (push jobs) | Requesters (accept inference requests) | +| Purpose | Schedules + load-balances jobs to live workers | Front-door for inference clients; creates JobRegistry entries | +| Worker env var | `WORKER_GATEWAY_URL` | (not used by worker) | You only interact with the Dispatcher if you're building an *inference client* (sending prompts to the network). For running a worker, you only need the gateway. diff --git a/docs/installation-linux.md b/docs/installation-linux.md index e250b16..c9af19d 100644 --- a/docs/installation-linux.md +++ b/docs/installation-linux.md @@ -7,7 +7,7 @@ Tested on **Ubuntu 24.04 LTS** with an NVIDIA RTX 4070. Should work identically | Tool | Version | Purpose | | ----------------------------- | ------- | --------------------------------------------- | | **Docker Engine** | ≥ 26.0 | Runs the worker container | -| **NVIDIA Container Toolkit** | latest | Only if you want Ollama in Docker (we won't) | +| **NVIDIA Container Toolkit** | latest | Only if you want Ollama in Docker (we won't) | | **Ollama** | ≥ 0.5 | Local LLM runtime | | **Foundry (cast)** | ≥ 1.7.1 | EVM wallet ops, contract reads | | **Bash** | ≥ 4.0 | Required for several scripts | @@ -122,7 +122,7 @@ sudo systemctl edit ollama Add: -``` +```ini [Service] Environment="OLLAMA_HOST=0.0.0.0:11434" ``` diff --git a/docs/installation-macos.md b/docs/installation-macos.md index ffd66f0..8670043 100644 --- a/docs/installation-macos.md +++ b/docs/installation-macos.md @@ -6,25 +6,25 @@ Tested on **macOS 14 Sonoma** (M2 Pro). Should work identically on macOS 13+ on Apple Silicon (M1/M2/M3/M4) does **not** have CUDA — Ollama uses Metal acceleration instead, which is automatic and fast. **Performance comparison for llama3-8b inference (rough):** -| Hardware | Tokens/sec | Job time (typical) | -| -------------------- | ----------- | ------------------ | -| M1 8 GB | ~10 | very slow | -| M1 Pro / M2 8 GB | ~20 | borderline | -| M2 Pro 16 GB | ~30-40 | usable | -| M3 Max / M4 Max | ~70-90 | comfortable | -| RTX 3060 (CUDA, 12GB)| ~80 | comfortable | -| RTX 4070 (CUDA, 12GB)| ~120 | competitive | +| Hardware | Tokens/sec | Job time (typical) | +| --------------------- | ----------- | ------------------ | +| M1 8 GB | ~10 | very slow | +| M1 Pro / M2 8 GB | ~20 | borderline | +| M2 Pro 16 GB | ~30-40 | usable | +| M3 Max / M4 Max | ~70-90 | comfortable | +| RTX 3060 (CUDA, 12GB) | ~80 | comfortable | +| RTX 4070 (CUDA, 12GB) | ~120 | competitive | If you're on an 8 GB Mac, you can run the worker but expect to lose timeouts on some jobs. **16 GB unified memory** is the practical floor for competitive throughput. ## Required software -| Tool | Version | Purpose | Install | -| ------------------ | ------- | ------------------------------------ | -------------------------------------- | -| **Docker Desktop** | ≥ 4.30 | Runs the worker container | `brew install --cask docker` | -| **Ollama** | ≥ 0.5 | Local LLM runtime | `brew install ollama` | +| Tool | Version | Purpose | Install | +| ------------------ | ------- | ------------------------------------ | ----------------------------------------------------------- | +| **Docker Desktop** | ≥ 4.30 | Runs the worker container | `brew install --cask docker` | +| **Ollama** | ≥ 0.5 | Local LLM runtime | `brew install ollama` | | **Foundry (cast)** | ≥ 1.7.1 | EVM wallet ops, contract reads | `curl -L https://foundry.paradigm.xyz \| bash && foundryup` | -| **Bash** | ≥ 4.0 | Required for several scripts | `brew install bash` | +| **Bash** | ≥ 4.0 | Required for several scripts | `brew install bash` | ## Install steps @@ -70,7 +70,7 @@ bash --version # GNU bash, version 5.x macOS hasn't shipped a newer bash since 2007 because bash 4+ is GPLv3 (Apple won't bundle GPLv3 code). Several scripts (e.g. `04-import-key.sh`, `06-fund-worker.sh`) use the `${var,,}` lowercase parameter expansion introduced in bash 4.0. On the stock bash you'll see: -``` +```text ./04-import-key.sh: line 35: ${reported,,}: bad substitution ``` diff --git a/docs/installation-windows.md b/docs/installation-windows.md index 75ffde3..dd7f816 100644 --- a/docs/installation-windows.md +++ b/docs/installation-windows.md @@ -4,12 +4,12 @@ Tested on **Windows 11** (build 26200, PowerShell 5.1+). Should work identically ## Required software -| Tool | Version | Purpose | Install | -| ------------------- | ------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------ | -| **Docker Desktop** | ≥ 4.30 | Runs the worker container | `winget install --id Docker.DockerDesktop --exact` | -| **Ollama** | ≥ 0.5 | Local LLM runtime | `winget install --id Ollama.Ollama --exact` | +| Tool | Version | Purpose | Install | +| ------------------- | ------------------- | -------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| **Docker Desktop** | ≥ 4.30 | Runs the worker container | `winget install --id Docker.DockerDesktop --exact` | +| **Ollama** | ≥ 0.5 | Local LLM runtime | `winget install --id Ollama.Ollama --exact` | | **Foundry (cast)** | ≥ 1.7.1 | EVM wallet ops, contract reads, on-chain sends | Download from [foundry-rs releases](https://github.com/foundry-rs/foundry/releases) | -| **WSL2** (optional) | latest | Only needed if you prefer Bash over PowerShell | `wsl --install` (built into Windows 10/11) | +| **WSL2** (optional) | latest | Only needed if you prefer Bash over PowerShell | `wsl --install` (built into Windows 10/11) | ## One-shot installer diff --git a/docs/onboarding.md b/docs/onboarding.md index 3beea59..38e58da 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -10,14 +10,14 @@ The toolkit's `00-*` through `08-*` scripts walk you through onboarding. This do ## Cost summary (mainnet) -| Item | Amount | Reversible? | -| ----------------------------- | --------------------------------- | ---------------------------------------------- | +| Item | Amount | Reversible? | +| ----------------------------- | --------------------------------- | -------------------------------------------------------------------- | | Stake | 50,000 LCAI | Yes — `deregister` returns it to worker wallet (minus any slashing). | -| Funding tx gas | < 0.01 LCAI | No (spent on gas). | -| Registration tx gas | < 0.05 LCAI | No. | -| Gas buffer in worker wallet | ~5 LCAI (recommended) | Yes — sweep when no longer needed. | -| Per-job ack + complete gas | < 0.001 LCAI per job | No. | -| **Total upfront** | **~50,005 LCAI** | | +| Funding tx gas | < 0.01 LCAI | No (spent on gas). | +| Registration tx gas | < 0.05 LCAI | No. | +| Gas buffer in worker wallet | ~5 LCAI (recommended) | Yes — sweep when no longer needed. | +| Per-job ack + complete gas | < 0.001 LCAI per job | No. | +| **Total upfront** | **~50,005 LCAI** | | ## Phase 00 — Generate a fresh worker key @@ -264,7 +264,7 @@ The full env block passed to the container includes all the values from `env.{ps ### Healthy startup looks like this -``` +```text INFO worker registration validated — on-chain key matches local key address=0x6BE4... WARN ollama model verification failed (non-fatal) error="missing models on Ollama server: [llama3-8b]" INFO blob mode: eip-4844 (beacon) diff --git a/docs/operations.md b/docs/operations.md index ac84300..ff5ae1a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -21,7 +21,7 @@ After [onboarding](./onboarding.md) is done, these are the things you'll actuall A successful job log block looks like this (timestamps and IDs vary): -``` +```text INFO ws_job_received jobId=543 INFO processing job jobID=543 model="f4a414fa..." sessionID=336 INFO stage 1: sending ack tx jobID=543 stage=ack timeout=15s diff --git a/docs/security.md b/docs/security.md index 43399b5..5d9118b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,14 +6,14 @@ How the toolkit handles keys, what your attack surface looks like, and the harde There are three classes of secret involved in running a worker. Their loss has very different consequences: -| Secret | What an attacker can do with it | Where the toolkit stores it | Practical risk tier | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | ------------------- | -| `FUNDER_PRIVKEY` | Drain your funder wallet — could be tens of thousands of LCAI or more. | **Not persisted by the toolkit.** Session env var only. | **Critical** | -| `WORKER_PRIVKEY` | Drain the worker wallet (rewards + ~5 LCAI gas buffer). Call `deregister` to release stake, then drain it too. Triggers a full lifecycle that takes minutes; you'd notice if you're watching. | `scripts//secrets.{ps1,env}`, ACL'd to current user. | **High** | -| `WORKER_PASSWORD` | Combined with the keystore file (`~/lightchain-worker/keys/eth-keystore/...`), an attacker can decrypt the worker privkey. Same impact as having `WORKER_PRIVKEY` directly. | Same file as `WORKER_PRIVKEY`. (Effectively redundant in our setup — see note below.) | **High** | +| Secret | What an attacker can do with it | Where the toolkit stores it | Practical risk tier | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ------------------- | +| `FUNDER_PRIVKEY` | Drain your funder wallet — could be tens of thousands of LCAI or more. | **Not persisted by the toolkit.** Session env var only. | **Critical** | +| `WORKER_PRIVKEY` | Drain the worker wallet (rewards + ~5 LCAI gas buffer). Call `deregister` to release stake, then drain it too. Triggers a full lifecycle that takes minutes; you'd notice if you're watching. | `scripts//secrets.{ps1,env}`, ACL'd to current user. | **High** | +| `WORKER_PASSWORD` | Combined with the keystore file (`~/lightchain-worker/keys/eth-keystore/...`), an attacker can decrypt the worker privkey. Same impact as having `WORKER_PRIVKEY` directly. | Same file as `WORKER_PRIVKEY`. (Effectively redundant in our setup — see note below.) | **High** | > **Note on `WORKER_PASSWORD` redundancy**: in the toolkit's default layout, `WORKER_PRIVKEY` and `WORKER_PASSWORD` live in the same file (`secrets.ps1` / `secrets.env`). Anyone who reads the file gets both. The password is only a defense-in-depth layer for the keystore file itself, useful if (and only if) the keystore leaks but `secrets.{ps1,env}` doesn't. - +> > **Why the worker key sits "in the open" at all**: the worker container has to sign on-chain txs (ack, complete) on every job, dozens of times per day. A hardware-wallet-mediated signing flow isn't feasible at that latency. So the privkey has to be reachable by an automated process. The Lightchain design accepts this and asks you to treat the worker key as **hot working capital** — fund it minimally, sweep rewards regularly to a cold wallet. ## What the toolkit does by default @@ -116,17 +116,23 @@ Without FDE, anyone with physical access can mount the disk in another machine a If you suspect the worker key has been compromised: 1. **Immediately deregister** — the on-chain `deregister` tx releases your stake back to the worker wallet. + ```powershell .\scripts\powershell\deregister.ps1 -Force ``` + 2. **Sweep the worker wallet to a fresh cold wallet** (NOT your normal funder if you suspect the funder is also compromised). + ```powershell .\scripts\powershell\sweep-rewards.ps1 -To 0xFreshColdWallet -GasBuffer 0 ``` + 3. **Stop the container.** + ```powershell .\scripts\powershell\stop.ps1 ``` + 4. Generate a new worker key, re-fund, re-register — but **on a different machine** if you believe the compromise was via the host. The reused machine is contaminated until you've rotated all secrets and reinstalled the OS. The deregister → sweep flow has a small race window: between the deregister tx confirming and your sweep tx confirming, the attacker can also try to sweep. In practice they'd need to be watching the chain in real-time AND have automation to react. If you're worried, send both txs from the same shell with the second one immediately ready to fire. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a27f9d7..d907d8f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -4,24 +4,24 @@ Every gotcha we hit setting up real workers, with the actual log lines and the a ## Quick triage -| Symptom | Jump to | -| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | -| Job arrives, dies at "stage 5 (resolve model)" | [Model hash mismatch](#1-model-hash-mismatch-stage-5-failure) — **the #1 issue** | -| Startup log: `ollama model verification failed (non-fatal)` | [Benign WARN](#2-warn-ollama-model-verification-failed) | -| Metrics show `worker_ollama_up = 0` forever | [Red-herring metric](#3-worker_ollama_up-stays-at-0) | -| No jobs ever arrive (hours of silence after `websocket connected`) | [Cold start vs real bug](#4-no-jobs-arriving) | -| Container restarts repeatedly | [Crash loop diagnosis](#5-container-restart-loop) | -| `Permission denied` pulling worker image | [Image is public — wrong URL](#6-image-pull-permission-denied) | -| `AccessDeniedException: 403` pulling genesis files / chain image | [You're running the wrong stack](#7-403-on-genesis-or-chain-node-image) | -| `failed to connect to the docker API at npipe:////./pipe/docker_engine` | [Docker Desktop not started](#8-docker-engine-not-running) | -| `host.docker.internal` resolves IPv6 only, Ollama unreachable from container | [Windows IPv6 quirk](#9-windows-host.docker.internal-ipv6-first) | -| `cast wallet new` fails, "command not found" | [Foundry not on PATH](#10-foundry-cast-not-on-path) | -| `register` reverts with `Pausable: paused` | [Protocol pause / wait](#11-registration-reverts-with-pausable-paused) | -| `register` reverts with `insufficient stake` | [Fund more LCAI](#12-registration-reverts-with-insufficient-stake) | -| WS reconnect every hour at the same minute | [Likely benign](#13-hourly-ws-reconnect) | -| Job arrives, dies at stage 6 with `connection refused` | [Ollama not running](#14-stage-6-connection-refused) | -| Job arrives, dies at stage 6 with `model not found` | [Wrong Ollama alias](#15-stage-6-model-not-found) | -| Job arrives, dies at stage 1 with `insufficient funds for gas` | [Worker wallet drained](#16-stage-1-insufficient-funds) | +| Symptom | Jump to | +| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| Job arrives, dies at "stage 5 (resolve model)" | [Model hash mismatch](#1-model-hash-mismatch-stage-5-failure) — **the #1 issue** | +| Startup log: `ollama model verification failed (non-fatal)` | [Benign WARN](#2-warn-ollama-model-verification-failed) | +| Metrics show `worker_ollama_up = 0` forever | [Red-herring metric](#3-worker_ollama_up-stays-at-0) | +| No jobs ever arrive (hours of silence after `websocket connected`) | [Cold start vs real bug](#4-no-jobs-arriving) | +| Container restarts repeatedly | [Crash loop diagnosis](#5-container-restart-loop) | +| `Permission denied` pulling worker image | [Image is public — wrong URL](#6-image-pull-permission-denied) | +| `AccessDeniedException: 403` pulling genesis files / chain image | [You're running the wrong stack](#7-403-on-genesis-or-chain-node-image) | +| `failed to connect to the docker API at npipe:////./pipe/docker_engine` | [Docker Desktop not started](#8-docker-engine-not-running) | +| `host.docker.internal` resolves IPv6 only, Ollama unreachable from container | [Windows IPv6 quirk](#9-windows-hostdockerinternal-ipv6-first) | +| `cast wallet new` fails, "command not found" | [Foundry not on PATH](#10-foundry-cast-not-on-path) | +| `register` reverts with `Pausable: paused` | [Protocol pause / wait](#11-registration-reverts-with-pausable-paused) | +| `register` reverts with `insufficient stake` | [Fund more LCAI](#12-registration-reverts-with-insufficient-stake) | +| WS reconnect every hour at the same minute | [Likely benign](#13-hourly-ws-reconnect) | +| Job arrives, dies at stage 6 with `connection refused` | [Ollama not running](#14-stage-6-connection-refused) | +| Job arrives, dies at stage 6 with `model not found` | [Wrong Ollama alias](#15-stage-6-model-not-found) | +| Job arrives, dies at stage 1 with `insufficient funds for gas` | [Worker wallet drained](#16-stage-1-insufficient-funds) | --- @@ -38,7 +38,7 @@ Every gotcha we hit setting up real workers, with the actual log lines and the a For mainnet `llama3-8b`: -``` +```text keccak256("llama3-8b") = 0xf4a414fa51803433e9197f32cda96d5cb2ac8269c481eb0262fe2dd11f428848 keccak256("llama3-8b:latest") = 0xbd1d7bf923df30d48c3259d08bd1308e13f4e6eac563abb2fe02ca84a608b194 keccak256("llama3:8b") = 0xa4dec912d6dd24b224db8b32c54cea79fc4c2208b29f6937679d17c34672741e @@ -73,7 +73,7 @@ This is what the toolkit defaults to. **Symptom:** at every container startup: -``` +```text WARN ollama model verification failed (non-fatal) error="missing models on Ollama server: [llama3-8b]" ``` @@ -92,7 +92,7 @@ If this bothers you, open a feature request on the Lightchain GitHub asking them **Symptom:** the metric never increments from its initial value, even hours into a running worker: -``` +```text # HELP worker_ollama_up 1 when the Ollama health-check returned 200, 0 otherwise (including check-itself errors). worker_ollama_up 0 ``` @@ -169,13 +169,13 @@ docker logs --previous lightchain-worker 2>&1 **Symptom:** -``` +```text Error response from daemon: Head ".../worker:latest": denied: Permission "artifactregistry.repositories.downloadArtifacts" denied ``` **Cause:** you tried to pull the **private** image. The public worker image lives at: -``` +```text us-central1-docker.pkg.dev/lightchain/lightchain-mainnet-public-docker/worker:latest ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ note the "public" in the path @@ -187,7 +187,7 @@ Anything without `public` in the repo name is internal-only and won't work from **Symptom:** -``` +```text AccessDeniedException: 403 does not have storage.objects.list access ... Error: Permission 'artifactregistry.repositories.downloadArtifacts' denied ``` @@ -200,7 +200,7 @@ Error: Permission 'artifactregistry.repositories.downloadArtifacts' denied **Symptom:** `docker ps` returns: -``` +```text error during connect: this error may indicate that the docker daemon is not running: Get "http://%2F%2F.%2Fpipe%2Fdocker_engine/...": open //./pipe/docker_engine: The system cannot find the file specified. @@ -242,7 +242,7 @@ The toolkit defaults to this on Windows. Note that `192.168.65.254` is Docker De **Symptom:** -``` +```text cast : The term 'cast' is not recognized as the name of a cmdlet, function, script file, or operable program. ``` @@ -260,7 +260,7 @@ Get-Command cast **Symptom:** Phase 07's `register` call returns: -``` +```text Error: server returned an error response: error code 3: execution reverted, data: "0x..." Pausable: paused / EnforcedPause ``` @@ -273,7 +273,7 @@ Pausable: paused / EnforcedPause **Symptom:** -``` +```text Error: ... reverted ... insufficient stake ``` @@ -294,7 +294,7 @@ $env:FUNDER_PRIVKEY = "0x..." **Symptom:** every hour, on a near-perfect cycle, you see: -``` +```text INFO reconciler: pass complete start=N safe_head=M chunks=K added=0 WARN websocket disconnected, reconnecting error="read: failed to get reader: failed to read frame header: EOF" INFO websocket connected to gateway @@ -308,7 +308,7 @@ INFO websocket connected to gateway **Symptom:** -``` +```text {"level":"ERROR","msg":"stage 6 failed","error":"connection refused: http://host.docker.internal:11434/api/generate"} ``` @@ -326,7 +326,7 @@ Verify with `Invoke-RestMethod http://127.0.0.1:11434/api/tags` (Windows) or `cu **Symptom:** -``` +```text {"level":"ERROR","msg":"stage 6 failed","error":"model \"llama3-8b\" not found"} ``` @@ -344,7 +344,7 @@ ollama list # confirm both 'llama3:8b' and 'llama3-8b:latest' appear **Symptom:** -``` +```text {"level":"ERROR","msg":"stage 1 (ack) failed","error":"insufficient funds for gas * price + value"} ```