diff --git a/.github/workflows/deploy-app-walrus.yml b/.github/workflows/deploy-app-walrus.yml index 29062c60..60d72aaa 100644 --- a/.github/workflows/deploy-app-walrus.yml +++ b/.github/workflows/deploy-app-walrus.yml @@ -30,7 +30,7 @@ jobs: uses: pnpm/action-setup@v4 - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Validate ws-resources.json exists run: | diff --git a/.github/workflows/release-oc-memwal.yml b/.github/workflows/release-oc-memwal.yml index 9994accc..ead6e1ce 100644 --- a/.github/workflows/release-oc-memwal.yml +++ b/.github/workflows/release-oc-memwal.yml @@ -40,7 +40,7 @@ jobs: run: npm install -g npm@latest - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Build SDK (dependency) run: pnpm build:sdk @@ -51,14 +51,61 @@ jobs: - name: Build plugin run: cd packages/openclaw-memory-memwal && npm run build - # ── main branch → stable release (latest) ── + # ── main branch → changeset version + stable release (latest) ── + - name: Apply changesets (update version & CHANGELOG) + if: github.ref == 'refs/heads/main' + id: changeset_version + run: | + pnpm changeset version 2>/dev/null || true + if git diff --quiet; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Commit changelog & version bump + if: github.ref == 'refs/heads/main' && steps.changeset_version.outputs.has_changes == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "chore: version packages & update changelog [skip ci]" + git push + - name: Publish stable release if: github.ref == 'refs/heads/main' + id: publish_oc run: | - BASE_VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") - npm view @mysten-incubation/oc-memwal@$BASE_VERSION version 2>/dev/null \ - && echo "Version $BASE_VERSION already published, skipping" && exit 0 + VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") + echo "version=$VERSION" >> $GITHUB_OUTPUT + npm view @mysten-incubation/oc-memwal@$VERSION version 2>/dev/null \ + && echo "Version $VERSION already published, skipping" && echo "published=false" >> $GITHUB_OUTPUT && exit 0 cd packages/openclaw-memory-memwal && npm publish --provenance --access public + echo "published=true" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + if: github.ref == 'refs/heads/main' && steps.publish_oc.outputs.published == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const version = '${{ steps.publish_oc.outputs.version }}'; + const tag = `@mysten-incubation/oc-memwal@${version}`; + let body = `Release @mysten-incubation/oc-memwal v${version}`; + try { + const changelog = fs.readFileSync('packages/openclaw-memory-memwal/CHANGELOG.md', 'utf8'); + const match = changelog.match(/## \d+\.\d+\.\d+[\s\S]*?(?=## \d+\.\d+\.\d+|$)/); + if (match) body = match[0].trim(); + } catch (e) { /* use default body */ } + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: tag, + body, + draft: false, + prerelease: false, + }); # ── staging branch → release candidate (rc tag, auto-increment) ── - name: Publish staging release candidate diff --git a/.github/workflows/release-sdk.yml b/.github/workflows/release-sdk.yml index 401414f5..ab452645 100644 --- a/.github/workflows/release-sdk.yml +++ b/.github/workflows/release-sdk.yml @@ -40,7 +40,7 @@ jobs: run: npm install -g npm@latest - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Typecheck run: pnpm --filter @mysten-incubation/memwal typecheck @@ -48,14 +48,64 @@ jobs: - name: Build SDK run: pnpm build:sdk - # ── main branch → stable release (latest) ── + # ── main branch → changeset version + stable release (latest) ── + - name: Apply changesets (update version & CHANGELOG) + if: github.ref == 'refs/heads/main' + id: changeset_version + run: | + # Consume pending changesets → bump version + update CHANGELOG.md + pnpm changeset version 2>/dev/null || true + # Check if changeset produced any changes + if git diff --quiet; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Commit changelog & version bump + if: github.ref == 'refs/heads/main' && steps.changeset_version.outputs.has_changes == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "chore: version packages & update changelog [skip ci]" + git push + - name: Publish stable release if: github.ref == 'refs/heads/main' + id: publish_sdk run: | - BASE_VERSION=$(node -p "require('./packages/sdk/package.json').version") - npm view @mysten-incubation/memwal@$BASE_VERSION version 2>/dev/null \ - && echo "Version $BASE_VERSION already published, skipping" && exit 0 + VERSION=$(node -p "require('./packages/sdk/package.json').version") + echo "version=$VERSION" >> $GITHUB_OUTPUT + npm view @mysten-incubation/memwal@$VERSION version 2>/dev/null \ + && echo "Version $VERSION already published, skipping" && echo "published=false" >> $GITHUB_OUTPUT && exit 0 cd packages/sdk && npm publish --provenance --access public + echo "published=true" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + if: github.ref == 'refs/heads/main' && steps.publish_sdk.outputs.published == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const version = '${{ steps.publish_sdk.outputs.version }}'; + const tag = `@mysten-incubation/memwal@${version}`; + // Extract latest changelog entry + let body = `Release @mysten-incubation/memwal v${version}`; + try { + const changelog = fs.readFileSync('packages/sdk/CHANGELOG.md', 'utf8'); + const match = changelog.match(/## \d+\.\d+\.\d+[\s\S]*?(?=## \d+\.\d+\.\d+|$)/); + if (match) body = match[0].trim(); + } catch (e) { /* use default body */ } + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: tag, + body, + draft: false, + prerelease: false, + }); # ── staging branch → release candidate (rc tag, auto-increment) ── - name: Publish staging release candidate diff --git a/README.md b/README.md index d482738f..28122bc3 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ retrieving them with semantic search. > MemWal is currently in beta and actively evolving. While fully usable today, we continue to refine the developer experience and operational guidance. We welcome feedback from early builders as we continue to improve the product. +## For AI Agents + +- **Single-file guide**: Read [`SKILL.md`](SKILL.md) for a complete integration reference (install, configure, API surface, troubleshooting) +- **LLM-friendly docs**: [`llms.txt`](https://docs.memwal.ai/llms.txt) — structured overview following the [llmstxt.org](https://llmstxt.org) standard +- **Full context**: [`llms-full.txt`](https://docs.memwal.ai/llms-full.txt) — expanded version with inlined page content + ## Install ```bash @@ -60,7 +66,13 @@ From the repository root: pnpm install ``` -Then start the surface you need, for example: +> **Important**: Build the SDK first — apps depend on its compiled output. + +```bash +pnpm build:sdk +``` + +Then start the surface you need: ```bash pnpm dev:app @@ -69,7 +81,7 @@ pnpm dev:chatbot pnpm dev:researcher ``` -For broader local setup guidance, see: +For the full step-by-step setup guide, see: - [Run the Repo Locally](docs/contributing/run-repo-locally.md) @@ -81,6 +93,18 @@ For broader local setup guidance, see: | `@mysten-incubation/memwal/manual` | Manual client flow (`MemWalManual`). You handle embedding calls and local SEAL operations. The relayer still handles upload relay, registration, search, and restore. | | `@mysten-incubation/memwal/ai` | Vercel AI SDK integration - wraps `MemWal` as middleware for use with `streamText`, `generateText`, etc. | +## OpenClaw / NemoClaw Plugin + +[`@mysten-incubation/oc-memwal`](packages/openclaw-memory-memwal) — a memory plugin for [OpenClaw](https://openclaw.ai) agents. It gives OpenClaw persistent, encrypted memory via MemWal with automatic recall and capture hooks. + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +- [Plugin Quick Start](docs/openclaw/quick-start.md) +- [How It Works](docs/openclaw/how-it-works.md) +- [Reference](docs/openclaw/reference.md) + ## How It Works 1. **Scope** - Each memory operation runs inside an `owner + namespace` boundary diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 00000000..00cad692 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,253 @@ +--- +name: memwal +version: 0.0.1 +description: | + Privacy-first AI memory SDK for decentralized storage on Sui blockchain with Walrus. + + Use when users say: + - "add memory to my app" + - "store encrypted memories" + - "integrate MemWal" + - "AI agent memory" + - "persistent memory SDK" + - "Walrus memory storage" + - "setup MemWal" + - "recall memories" + +keywords: + - memwal + - memory sdk + - ai memory + - encrypted memory + - walrus storage + - sui blockchain + - delegate key + - semantic search + - vercel ai sdk +--- + +# MemWal — Privacy-First AI Memory SDK + +MemWal is a TypeScript SDK for persistent, encrypted AI memory. It stores memories on Walrus (decentralized storage), encrypts them with SEAL, enforces ownership onchain via Sui smart contracts, and retrieves them with semantic (vector) search. Memories are scoped by `owner + namespace` — each namespace is an isolated memory space. + +--- + +## When to Use + +Use MemWal when your app or agent needs: + +- **Persistent memory** across sessions, devices, or restarts +- **Encrypted storage** — end-to-end encryption, only the owner and authorized delegates can decrypt +- **Semantic recall** — retrieve memories by meaning, not just keywords +- **Decentralized storage** — no single point of failure, stored on Walrus +- **Onchain ownership** — cryptographically enforced access control on Sui +- **Cross-app memory** — share memory between apps via delegate keys + +--- + +## When NOT to Use + +- Temporary conversation context that only matters in the current session +- Large file storage (MemWal is optimized for text memories) +- Use cases that don't need encryption or decentralization + +--- + +## Installation + +```bash +# Install the SDK +pnpm add @mysten-incubation/memwal + +# Optional: for Vercel AI SDK integration +pnpm add ai zod + +# Optional: for manual client (client-side SEAL encryption) +pnpm add @mysten/sui @mysten/seal @mysten/walrus +``` + +--- + +## Quick Start + +### 1. Get Your Credentials + +You need a **delegate key** (Ed25519 private key) and **account ID** (MemWalAccount object ID on Sui). + +Generate them at: +- Production: https://memwal.ai or https://memwal.wal.app +- Staging: https://staging.memwal.ai + +### 2. Initialize the SDK + +```ts +import { MemWal } from "@mysten-incubation/memwal"; + +const memwal = MemWal.create({ + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "my-app", +}); +``` + +### 3. Store and Recall Memories + +```ts +// Store a memory +await memwal.remember("User prefers dark mode and works in TypeScript."); + +// Recall by meaning +const result = await memwal.recall("What are the user's preferences?"); +console.log(result.results); + +// Extract and store facts from text +await memwal.analyze("I live in Hanoi and prefer dark mode."); + +// Check relayer health +await memwal.health(); +``` + +--- + +## SDK Entry Points + +| Entry Point | Import | Description | +|---|---|---| +| `MemWal` | `@mysten-incubation/memwal` | **Default.** Relayer handles embedding, SEAL encryption, Walrus upload, vector search | +| `MemWalManual` | `@mysten-incubation/memwal/manual` | Manual flow — client handles embedding and SEAL encryption | +| `withMemWal` | `@mysten-incubation/memwal/ai` | Vercel AI SDK middleware — auto recall + save around AI conversations | +| Account utils | `@mysten-incubation/memwal/account` | Account creation, delegate key management | + +--- + +## API Surface + +### MemWal Methods + +| Method | Description | Returns | +|---|---|---| +| `remember(text, namespace?)` | Store one memory (relayer embeds, encrypts, uploads) | `{ id, blob_id, owner, namespace }` | +| `recall(query, limit?, namespace?)` | Semantic search for memories | `{ results: [{ blob_id, text, distance }], total }` | +| `analyze(text, namespace?)` | Extract facts via LLM, store each as a memory | `{ facts: [{ text, id, blob_id }], total, owner }` | +| `restore(namespace, limit?)` | Rebuild missing index entries from Walrus | `{ restored, skipped, total, namespace, owner }` | +| `health()` | Check relayer health | `{ status, version }` | +| `getPublicKeyHex()` | Get hex-encoded public key | `string` | + +### Lower-Level Methods + +| Method | Description | +|---|---| +| `rememberManual({ blobId, vector, namespace? })` | Register pre-uploaded blob with pre-computed vector | +| `recallManual({ vector, limit?, namespace? })` | Search with pre-computed vector (returns blob IDs only) | +| `embed(text)` | Generate embedding vector (no storage) | + +--- + +## Configuration + +### MemWalConfig + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `key` | `string` | Yes | — | Ed25519 delegate private key in hex | +| `accountId` | `string` | Yes | — | MemWalAccount object ID on Sui | +| `serverUrl` | `string` | No | `http://localhost:8000` | Relayer URL | +| `namespace` | `string` | No | `"default"` | Default namespace for memory isolation | + +### Managed Relayer Endpoints + +| Network | Relayer URL | +|---|---| +| **Production** (mainnet) | `https://relayer.memwal.ai` | +| **Staging** (testnet) | `https://relayer.staging.memwal.ai` | + +--- + +## Vercel AI SDK Integration + +```ts +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { withMemWal } from "@mysten-incubation/memwal/ai"; + +const model = withMemWal(openai("gpt-4o"), { + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "chat", + maxMemories: 5, + autoSave: true, + minRelevance: 0.3, +}); + +const result = streamText({ + model, + messages: [{ role: "user", content: "What do you remember about me?" }], +}); +``` + +The middleware automatically: +- Recalls relevant memories before generation +- Extracts and saves facts from conversations after generation + +--- + +## OpenClaw / NemoClaw Plugin + +For OpenClaw agent integration, use the `@mysten-incubation/oc-memwal` plugin. + +### Install + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +### Configure + +Add to `~/.openclaw/openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "oc-memwal" }, + "entries": { + "oc-memwal": { + "enabled": true, + "config": { + "privateKey": "${MEMWAL_PRIVATE_KEY}", + "accountId": "0x...", + "serverUrl": "https://relayer.memwal.ai" + } + } + } + } +} +``` + +Lifecycle hooks run automatically: +- `before_prompt_build` — injects relevant memories as context +- `before_reset` — saves session summary +- `agent_end` — captures last response + +--- + +## Troubleshooting + +| Symptom | Fix | +|---|---| +| `health()` returns error | Check relayer URL is correct and reachable | +| `recall()` returns empty | Verify namespace matches what was used in `remember()` | +| `401 Unauthorized` | Verify delegate key is correct and registered on the account | +| SDK import errors | Run `pnpm add @mysten-incubation/memwal` — check Node.js ≥ 18 | +| Manual client errors | Install peer deps: `@mysten/sui @mysten/seal @mysten/walrus` | + +--- + +## Links + +- **Docs**: https://docs.memwal.ai +- **SDK on npm**: https://www.npmjs.com/package/@mysten-incubation/memwal +- **GitHub**: https://github.com/CommandOSSLabs/MemWal +- **Dashboard**: https://memwal.ai +- **llms.txt**: https://docs.memwal.ai/llms.txt diff --git a/apps/app/Dockerfile b/apps/app/Dockerfile index 593499d0..6b19f37d 100644 --- a/apps/app/Dockerfile +++ b/apps/app/Dockerfile @@ -18,7 +18,7 @@ COPY packages/sdk/package.json ./packages/sdk/ COPY apps/app/package.json ./apps/app/ # Install all workspace deps -RUN pnpm install +RUN pnpm install --frozen-lockfile # Build SDK first (apps/app depends on @mysten-incubation/memwal via workspace:*) COPY packages/sdk/ ./packages/sdk/ diff --git a/apps/researcher/Dockerfile b/apps/researcher/Dockerfile index 221ea3fd..9640bc8d 100644 --- a/apps/researcher/Dockerfile +++ b/apps/researcher/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /app COPY apps/researcher/package.json ./ # Install deps — @mysten-incubation/memwal is now on npm, no local SDK needed -RUN bun install +RUN bun install --frozen-lockfile # ── Stage 2: Build ───────────────────────────────────────── FROM oven/bun:1 AS builder diff --git a/apps/researcher/package.json b/apps/researcher/package.json index 0217aa35..517ebec7 100644 --- a/apps/researcher/package.json +++ b/apps/researcher/package.json @@ -24,7 +24,7 @@ "@ai-sdk/openai": "^3.0.41", "@ai-sdk/provider": "^3.0.3", "@ai-sdk/react": "3.0.39", - "@mysten-incubation/memwal": "^0.0.1-dev.0", + "@mysten-incubation/memwal": "0.0.1", "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", "@codemirror/state": "^6.5.0", diff --git a/docs/contributing/run-repo-locally.md b/docs/contributing/run-repo-locally.md index 7d5bae4e..ce3b0773 100644 --- a/docs/contributing/run-repo-locally.md +++ b/docs/contributing/run-repo-locally.md @@ -1,40 +1,131 @@ --- title: "Run the Repo Locally" +description: "Step-by-step guide to set up the MemWal monorepo for local development." --- -This monorepo contains: +## Prerequisites -- TypeScript applications under `apps/` -- the SDK under `packages/sdk` -- Rust backend services under `services/` -- Mintlify docs under `docs/` +| Tool | Version | Check | +|------|---------|-------| +| **Node.js** | ≥ 20 | `node -v` | +| **pnpm** | ≥ 9.12 | `pnpm -v` | +| **Rust** | latest stable (only for backend services) | `rustc --version` | -## Common Local Entry Points + +If you only work on TypeScript apps or docs, you don't need Rust. + -From the repository root: +## Step 1 — Clone and Install ```bash +git clone https://github.com/CommandOSSLabs/MemWal.git +cd MemWal pnpm install +``` + +## Step 2 — Build the SDK First + + +The apps depend on the SDK's compiled output. If you skip this step, apps will fail to start with import errors. + + +```bash +pnpm build:sdk +``` + +This compiles `packages/sdk` → `packages/sdk/dist/`. The apps import from `@mysten-incubation/memwal`, which resolves to this compiled output via the workspace. + +## Step 3 — Run What You Need + +Run individual surfaces from the repository root: + +```bash +# Docs site (Mintlify) pnpm dev:docs -pnpm dev:app -pnpm dev:noter -pnpm dev:chatbot -pnpm dev:researcher + +# Demo apps (pick one) +pnpm dev:app # Playground dashboard +pnpm dev:noter # Note-taking app +pnpm dev:chatbot # AI chatbot +pnpm dev:researcher # Research assistant + +# SDK in watch mode (recompiles on changes) +pnpm dev:sdk ``` -Backend services are run from their respective Rust service directories. +## Step 4 — Backend Services (Optional) -## Service Dependencies +The TypeScript apps talk to a managed relayer by default. You only need to run backend services if you're working on the relayer or indexer. -For relayer-oriented local work you will typically need: +### Relayer (`services/server`) -- PostgreSQL +Requires: +- PostgreSQL with `pgvector` extension - Sui RPC access - Walrus endpoints -- embedding provider credentials +- Embedding provider credentials (OpenAI-compatible) + +Quick start: + +```bash +# Start PostgreSQL with pgvector +docker compose -f services/server/docker-compose.yml up -d postgres + +# Configure environment +cp services/server/.env.example services/server/.env +# Edit .env with your credentials + +# Install sidecar dependencies +cd services/server/scripts && npm ci && cd .. + +# Run the relayer +cargo run +``` + +For the full relayer setup guide, see [Self-Hosting](/relayer/self-hosting). + +### Indexer (`services/indexer`) + +```bash +cd services/indexer +cargo run +``` + +The indexer polls Sui events and syncs account data into PostgreSQL. + +## Monorepo Structure + +``` +MemWal/ +├── packages/ +│ ├── sdk/ # @mysten-incubation/memwal — TypeScript SDK +│ └── openclaw-memory-memwal/ # @mysten-incubation/oc-memwal — OpenClaw plugin +├── apps/ +│ ├── app/ # Playground dashboard +│ ├── chatbot/ # AI chatbot demo +│ ├── noter/ # Note-taking demo +│ └── researcher/ # Research assistant demo +├── services/ +│ ├── server/ # Rust relayer (Axum) +│ ├── indexer/ # Rust Sui event indexer +│ └── contract/ # Move smart contract +├── docs/ # Mintlify documentation site +└── SKILL.md # Agent-first integration guide +``` + +## Troubleshooting -## Relayer Setup +| Problem | Cause | Fix | +|---------|-------|-----| +| `Cannot find module '@mysten-incubation/memwal'` | SDK not built | Run `pnpm build:sdk` first | +| `ERR_MODULE_NOT_FOUND` in apps | Stale SDK build | Run `pnpm build:sdk` again | +| `pnpm install` fails | Wrong pnpm version | Use pnpm ≥ 9.12: `corepack enable && corepack prepare pnpm@9.12.3 --activate` | +| Docs site won't start | Missing Mintlify | Run `pnpm install` from the root | +| Relayer crashes on boot | Missing pgvector | Install the `pgvector` PostgreSQL extension | +| Sidecar timeout | Missing sidecar deps | Run `cd services/server/scripts && npm ci` | -If you want to run the backend locally, start with the Relayer docs: +## See Also -- [Self-Hosting](/relayer/self-hosting) +- [Run Docs Locally](/contributing/run-docs-locally) — just the docs site +- [Self-Hosting](/relayer/self-hosting) — full relayer deployment +- [Environment Variables](/reference/environment-variables) — relayer configuration diff --git a/docs/docs.json b/docs/docs.json index a9c1044a..aed1fa5c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -84,7 +84,8 @@ "sdk/usage/with-memwal" ] }, - "sdk/api-reference" + "sdk/api-reference", + "sdk/changelog" ] } ] @@ -138,7 +139,8 @@ "openclaw/overview", "openclaw/quick-start", "openclaw/how-it-works", - "openclaw/reference" + "openclaw/reference", + "openclaw/changelog" ] } ] diff --git a/docs/llms-full.txt b/docs/llms-full.txt new file mode 100644 index 00000000..89a78093 --- /dev/null +++ b/docs/llms-full.txt @@ -0,0 +1,452 @@ +# MemWal + +> MemWal is a privacy-first AI memory layer. It stores encrypted memories on Walrus (decentralized storage) and retrieves them with semantic search. Ownership is enforced onchain via Sui smart contracts. The TypeScript SDK (`@mysten-incubation/memwal`) gives any app persistent, encrypted memory in a few lines of code. + +Important notes: + +- MemWal is currently in beta +- The SDK talks to a relayer service that handles embedding, SEAL encryption, Walrus upload/download, and vector search +- All content is end-to-end encrypted — only the owner and authorized delegates can decrypt +- Delegate keys provide scoped access — agents can read/write memory without holding the owner's private key +- Memory is scoped by `owner + namespace` — each namespace is an isolated memory space + +--- + +## Quick Start + +### Installation + +```bash +pnpm add @mysten-incubation/memwal +``` + +Optional peer dependencies: + +```bash +# For Vercel AI SDK middleware +pnpm add ai zod + +# For manual client (client-side SEAL encryption) +pnpm add @mysten/sui @mysten/seal @mysten/walrus +``` + +### Prerequisites + +- Node.js v18+ or Bun v1+ +- A delegate key and account ID (generate at https://memwal.ai or https://memwal.wal.app) +- A relayer URL (use `https://relayer.memwal.ai` for production or `https://relayer.staging.memwal.ai` for staging) + +### First Memory + +```ts +import { MemWal } from "@mysten-incubation/memwal"; + +const memwal = MemWal.create({ + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "demo", +}); + +await memwal.health(); +await memwal.remember("I live in Hanoi and prefer dark mode."); + +const result = await memwal.recall("What do we know about this user?"); +console.log(result.results); +``` + +--- + +## SDK Entry Points + +| Entry Point | Import | When to Use | +|---|---|---| +| `MemWal` | `@mysten-incubation/memwal` | **Recommended default** — relayer handles embeddings, SEAL, and storage | +| `MemWalManual` | `@mysten-incubation/memwal/manual` | Client-managed embeddings and local SEAL operations | +| `withMemWal` | `@mysten-incubation/memwal/ai` | Vercel AI SDK middleware — auto recall + save | +| Account utils | `@mysten-incubation/memwal/account` | Account creation, delegate key management | + +--- + +## SDK API Reference + +### `MemWal.create(config)` + +```ts +MemWal.create(config: MemWalConfig): MemWal +``` + +Config: + +| Property | Type | Required | Default | Notes | +|---|---|---|---|---| +| `key` | `string` | Yes | — | Ed25519 delegate private key in hex | +| `accountId` | `string` | Yes | — | MemWalAccount object ID on Sui | +| `serverUrl` | `string` | No | `http://localhost:8000` | Relayer URL | +| `namespace` | `string` | No | `"default"` | Default namespace for memory isolation | + +### `remember(text, namespace?): Promise` + +Store one memory through the relayer. The relayer handles embedding, SEAL encryption, Walrus upload, and vector indexing. + +Returns: +```ts +{ + id: string; // UUID for this entry + blob_id: string; // Walrus blob ID + owner: string; // Owner Sui address + namespace: string; // Namespace used +} +``` + +### `recall(query, limit?, namespace?): Promise` + +Search for memories matching a natural language query, scoped to `owner + namespace`. + +- `limit` defaults to `10` + +Returns: +```ts +{ + results: Array<{ + blob_id: string; // Walrus blob ID + text: string; // Decrypted plaintext + distance: number; // Cosine distance (lower = more similar) + }>; + total: number; +} +``` + +### `analyze(text, namespace?): Promise` + +Extract memorable facts from text using an LLM, then store each fact as a separate memory. + +Returns: +```ts +{ + facts: Array<{ + text: string; // Extracted fact + id: string; // UUID + blob_id: string; // Walrus blob ID + }>; + total: number; + owner: string; +} +``` + +### `restore(namespace, limit?): Promise` + +Rebuild missing indexed entries for one namespace from Walrus. Incremental — only re-indexes blobs that aren't already in the local database. + +- `limit` defaults to `50` + +Returns: +```ts +{ + restored: number; // Entries newly indexed + skipped: number; // Entries already in DB + total: number; // Total blobs found on-chain + namespace: string; + owner: string; +} +``` + +### `health(): Promise` + +Check relayer health. Does not require authentication. + +Returns: `{ status: string, version: string }` + +### `getPublicKeyHex(): Promise` + +Return the hex-encoded public key for the current delegate key. + +### Lower-level methods + +| Method | Description | +|--------|-------------| +| `rememberManual({ blobId, vector, namespace? })` | Register a pre-uploaded blob ID with a pre-computed vector | +| `recallManual({ vector, limit?, namespace? })` | Search with a pre-computed query vector (returns blob IDs, no decryption) | +| `embed(text)` | Generate an embedding vector for text (no storage) | + +--- + +## MemWalManual + +```ts +import { MemWalManual } from "@mysten-incubation/memwal/manual"; +``` + +Manual client flow — embed locally, SEAL encrypt locally, send encrypted payload + vector to relayer. + +### Config (extends MemWalConfig) + +| Field | Required | Notes | +|---|---|---| +| `embeddingApiKey` | Yes | OpenAI/OpenRouter-compatible embedding key | +| `embeddingApiBase` | No | Default: `https://api.openai.com/v1` | +| `embeddingModel` | No | Default: `text-embedding-3-small` | +| `packageId` | Yes | MemWal package ID on Sui | +| `suiPrivateKey` or `walletSigner` | One required | Local keypair or connected wallet | +| `suiNetwork` | No | Default: `mainnet` | + +--- + +## withMemWal (AI Middleware) + +```ts +import { withMemWal } from "@mysten-incubation/memwal/ai"; +``` + +Wraps a Vercel AI SDK model with automatic memory recall and save. + +```ts +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { withMemWal } from "@mysten-incubation/memwal/ai"; + +const model = withMemWal(openai("gpt-4o"), { + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "chat", + maxMemories: 5, + autoSave: true, + minRelevance: 0.3, +}); + +const result = streamText({ + model, + messages: [{ role: "user", content: "What do you remember about me?" }], +}); +``` + +**Before generation:** +- Reads the last user message +- Runs `recall()` against MemWal +- Filters by minimum relevance (`minRelevance`, default `0.3`) +- Injects matching memories into the prompt as a system message + +**After generation:** +- Optionally runs `analyze()` on the user message (fire-and-forget) +- Saves extracted facts asynchronously + +Options (extends MemWalConfig): + +| Option | Default | Description | +|--------|---------|-------------| +| `maxMemories` | `5` | Max memories to inject per request | +| `autoSave` | `true` | Auto-save new facts from conversation | +| `minRelevance` | `0.3` | Minimum similarity score (0–1) to include a memory | +| `debug` | `false` | Enable debug logging | + +--- + +## Account Management + +```ts +import { + createAccount, + addDelegateKey, + removeDelegateKey, + generateDelegateKey, +} from "@mysten-incubation/memwal/account"; +``` + +| Function | Description | +|----------|-------------| +| `generateDelegateKey()` | Generate a new Ed25519 keypair (returns `privateKey`, `publicKey`, `suiAddress`) | +| `createAccount(opts)` | Create a new MemWalAccount on-chain (one per Sui address) | +| `addDelegateKey(opts)` | Add a delegate key to an account (owner only) | +| `removeDelegateKey(opts)` | Remove a delegate key from an account (owner only) | + +--- + +## Utility Functions + +```ts +import { delegateKeyToSuiAddress, delegateKeyToPublicKey } from "@mysten-incubation/memwal"; +``` + +| Function | Description | +|----------|-------------| +| `delegateKeyToSuiAddress(privateKeyHex)` | Derive the Sui address from a delegate private key | +| `delegateKeyToPublicKey(privateKeyHex)` | Get the 32-byte public key from a delegate private key | + +--- + +## Configuration Reference + +### MemWalConfig + +Used by `MemWal.create(config)` and `withMemWal(model, options)`. + +| Field | Required | Notes | +|---|---|---| +| `key` | yes | Delegate private key in hex | +| `accountId` | yes | MemWalAccount object ID on Sui | +| `serverUrl` | no | Relayer URL. Default: `http://localhost:8000` | +| `namespace` | no | Default memory boundary. Default: `"default"` | + +### MemWalManualConfig + +Used by `MemWalManual.create(config)`. + +| Field | Required | Notes | +|---|---|---| +| `key` | yes | Delegate private key in hex | +| `serverUrl` | no | Relayer URL | +| `embeddingApiKey` | yes | OpenAI/OpenRouter-compatible embedding key | +| `embeddingApiBase` | no | Default: `https://api.openai.com/v1` | +| `embeddingModel` | no | Default: `text-embedding-3-small` | +| `packageId` | yes | MemWal package ID on Sui | +| `accountId` | yes | MemWalAccount object ID | +| `namespace` | no | Default namespace | +| `suiPrivateKey` | one of two | Use for local signing | +| `walletSigner` | one of two | Use a connected browser wallet instead | +| `suiNetwork` | no | `testnet` or `mainnet`. Default: `mainnet` | + +### Managed Relayer Endpoints + +| Network | Relayer URL | +|---|---| +| **Production** (mainnet) | `https://relayer.memwal.ai` | +| **Staging** (testnet) | `https://relayer.staging.memwal.ai` | + +--- + +## Relayer Overview + +The relayer is the backend that turns SDK calls into memory operations: + +- **Authenticates requests** by verifying Ed25519 signatures against onchain delegate keys +- **Generates embeddings** using an OpenAI-compatible API (default: `text-embedding-3-small`, 1536 dimensions) +- **Encrypts and decrypts** data through the SEAL sidecar +- **Uploads and downloads** encrypted blobs to/from Walrus +- **Stores and searches vectors** in PostgreSQL (pgvector) +- **Orchestrates flows** like `analyze` (LLM fact extraction) and `ask` (memory-augmented Q&A) +- **Restores memory spaces** by querying onchain blobs, decrypting, and re-indexing + +### Authentication Headers + +| Header | Description | +|--------|-------------| +| `x-public-key` | Hex-encoded Ed25519 public key (32 bytes) | +| `x-signature` | Hex-encoded Ed25519 signature (64 bytes) | +| `x-timestamp` | Unix timestamp in seconds (5-minute validity window) | +| `x-account-id` | Optional — MemWalAccount object ID hint | + +Signature format: `{timestamp}.{method}.{path}.{body_sha256}` + +### Relayer API Routes + +| Method | Route | Description | +|--------|-------|-------------| +| `GET` | `/health` | Service health check (no auth) | +| `POST` | `/api/remember` | Store text as encrypted memory | +| `POST` | `/api/recall` | Semantic search for memories | +| `POST` | `/api/remember/manual` | Register client-encrypted payload | +| `POST` | `/api/recall/manual` | Search with pre-computed vector | +| `POST` | `/api/analyze` | Extract facts via LLM, store each | +| `POST` | `/api/ask` | Memory-augmented Q&A | +| `POST` | `/api/restore` | Rebuild missing index entries | + +--- + +## OpenClaw / NemoClaw Plugin + +### Installation + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +### Configuration + +Set the delegate key as an environment variable: + +```bash +export MEMWAL_PRIVATE_KEY="your-64-char-hex-key" +``` + +Add to `~/.openclaw/openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "oc-memwal" }, + "entries": { + "oc-memwal": { + "enabled": true, + "config": { + "privateKey": "${MEMWAL_PRIVATE_KEY}", + "accountId": "0x...", + "serverUrl": "https://relayer.memwal.ai" + } + } + } + } +} +``` + +### Plugin Options + +| Option | Default | Description | +|--------|---------|-------------| +| `autoRecall` | `true` | Inject relevant memories before each turn | +| `autoCapture` | `true` | Extract and store facts after each turn | +| `maxRecallResults` | `5` | Max memories injected per turn | +| `minRelevance` | `0.3` | Relevance threshold (0-1) | +| `captureMaxMessages` | `10` | Messages to analyze for facts | +| `defaultNamespace` | `"default"` | Memory scope for the main agent | + +### Lifecycle Hooks + +| Hook | Trigger | What Happens | +|------|---------|--------------| +| `before_prompt_build` | Every LLM call | Relevant memories injected as context | +| `before_reset` | Before `/reset` | Session summary saved | +| `agent_end` | Agent finishes | Last response captured | + +--- + +## Environment Variables (Self-Hosting) + +### Required + +| Variable | Notes | +|---|---| +| `DATABASE_URL` | PostgreSQL connection string. `pgvector` must already exist | +| `MEMWAL_PACKAGE_ID` | Sui package ID | +| `MEMWAL_REGISTRY_ID` | Onchain registry object ID | +| `SEAL_KEY_SERVERS` | Comma-separated SEAL key server object IDs | + +### Usually Required + +| Variable | Notes | +|---|---| +| `SERVER_SUI_PRIVATE_KEY` | Primary server key | +| `OPENAI_API_KEY` | Embedding and fact-extraction provider | + +### Package Contract IDs + +Staging (Testnet): +``` +MEMWAL_PACKAGE_ID=0xcf6ad755a1cdff7217865c796778fabe5aa399cb0cf2eba986f4b582047229c6 +MEMWAL_REGISTRY_ID=0xe80f2feec1c139616a86c9f71210152e2a7ca552b20841f2e192f99f75864437 +``` + +Production (Mainnet): +``` +MEMWAL_PACKAGE_ID=0xcee7a6fd8de52ce645c38332bde23d4a30fd9426bc4681409733dd50958a24c6 +MEMWAL_REGISTRY_ID=0x0da982cefa26864ae834a8a0504b904233d49e20fcc17c373c8bed99c75a7edd +``` + +--- + +## Links + +- **Docs**: https://docs.memwal.ai +- **SDK on npm**: https://www.npmjs.com/package/@mysten-incubation/memwal +- **GitHub**: https://github.com/CommandOSSLabs/MemWal +- **Dashboard**: https://memwal.ai diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 00000000..e7bcd343 --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,58 @@ +# MemWal + +> MemWal is a privacy-first AI memory layer. It stores encrypted memories on Walrus (decentralized storage) and retrieves them with semantic search. Ownership is enforced onchain via Sui smart contracts. The TypeScript SDK (`@mysten-incubation/memwal`) gives any app persistent, encrypted memory in a few lines of code. + +Important notes: + +- MemWal is currently in beta +- The SDK talks to a relayer service that handles embedding, SEAL encryption, Walrus upload/download, and vector search +- All content is end-to-end encrypted — only the owner and authorized delegates can decrypt +- Delegate keys provide scoped access — agents can read/write memory without holding the owner's private key +- Memory is scoped by `owner + namespace` — each namespace is an isolated memory space + +## Docs + +- [What is MemWal?](https://docs.memwal.ai/getting-started/what-is-memwal): Overview of features, architecture, and use cases +- [Quick Start](https://docs.memwal.ai/getting-started/quick-start): Install the SDK, generate credentials, store and recall your first memory +- [Choose Your Path](https://docs.memwal.ai/getting-started/choose-your-path): Pick the right integration path (MemWal, MemWalManual, or withMemWal AI middleware) + +## SDK + +- [SDK Quick Start](https://docs.memwal.ai/sdk/quick-start): Install, configure, and store your first memory +- [MemWal Usage](https://docs.memwal.ai/sdk/usage/memwal): Default relayer-handled client — remember, recall, analyze, restore +- [MemWalManual Usage](https://docs.memwal.ai/sdk/usage/memwal-manual): Manual client — client-side embedding and SEAL encryption +- [withMemWal AI Middleware](https://docs.memwal.ai/sdk/usage/with-memwal): Vercel AI SDK integration — automatic memory recall and save +- [SDK API Reference](https://docs.memwal.ai/sdk/api-reference): Full method signatures, return types, and config fields + +## Relayer + +- [Relayer Overview](https://docs.memwal.ai/relayer/overview): Architecture, trust boundary, key pool, single-instance design +- [Public Relayer](https://docs.memwal.ai/relayer/public-relayer): Managed relayer endpoints provided by Walrus Foundation +- [Self-Hosting](https://docs.memwal.ai/relayer/self-hosting): Run your own relayer with full control over encryption and embedding +- [Relayer API Reference](https://docs.memwal.ai/relayer/api-reference): HTTP routes, authentication headers, request/response shapes + +## Smart Contract + +- [Contract Overview](https://docs.memwal.ai/contract/overview): Onchain ownership model and package IDs +- [Delegate Key Management](https://docs.memwal.ai/contract/delegate-key-management): Add, remove, and rotate delegate keys +- [Ownership & Permissions](https://docs.memwal.ai/contract/ownership-and-permissions): Access control model + +## OpenClaw Plugin + +- [OpenClaw Overview](https://docs.memwal.ai/openclaw/overview): NemoClaw/OpenClaw memory plugin overview +- [OpenClaw Quick Start](https://docs.memwal.ai/openclaw/quick-start): Install and configure the plugin +- [How It Works](https://docs.memwal.ai/openclaw/how-it-works): Architecture, hooks, and message flow +- [OpenClaw Reference](https://docs.memwal.ai/openclaw/reference): Hooks, tools, CLI, and configuration + +## Reference + +- [Configuration](https://docs.memwal.ai/reference/configuration): MemWalConfig, MemWalManualConfig, WithMemWalOptions +- [Environment Variables](https://docs.memwal.ai/reference/environment-variables): Relayer env vars for self-hosting + +## Optional + +- [Concepts: Memory Space](https://docs.memwal.ai/fundamentals/concepts/memory-space): Namespace isolation and memory boundaries +- [Architecture: Core Components](https://docs.memwal.ai/fundamentals/architecture/core-components): System overview and component responsibilities +- [Data Flow & Security Model](https://docs.memwal.ai/fundamentals/architecture/data-flow-security-model): Trust boundaries and encryption flows +- [Contributing: Run Repo Locally](https://docs.memwal.ai/contributing/run-repo-locally): Monorepo setup for contributors +- [Example Apps](https://docs.memwal.ai/examples/example-apps): Playground, Chatbot, Noter, Researcher diff --git a/docs/openclaw/changelog.md b/docs/openclaw/changelog.md new file mode 100644 index 00000000..1e3a3ce3 --- /dev/null +++ b/docs/openclaw/changelog.md @@ -0,0 +1,19 @@ +--- +title: "Changelog" +description: "Release history for the MemWal OpenClaw plugin." +--- + +Track what's new, changed, and fixed in `@mysten-incubation/oc-memwal`. + +For the latest version, see the [npm package page](https://www.npmjs.com/package/@mysten-incubation/oc-memwal). + +## 0.0.1 + +### Initial Release + +- NemoClaw/OpenClaw memory plugin powered by MemWal +- Automatic memory recall via `before_prompt_build` hook +- Automatic fact capture via `agent_end` hook +- Session summary on `before_reset` hook +- CLI commands: `openclaw memwal stats`, `openclaw memwal search` +- LLM tools: `memory_search`, `memory_store` diff --git a/docs/sdk/changelog.md b/docs/sdk/changelog.md new file mode 100644 index 00000000..b651ff12 --- /dev/null +++ b/docs/sdk/changelog.md @@ -0,0 +1,19 @@ +--- +title: "Changelog" +description: "Release history for the MemWal TypeScript SDK." +--- + +Track what's new, changed, and fixed in `@mysten-incubation/memwal`. + +For the latest version, see the [npm package page](https://www.npmjs.com/package/@mysten-incubation/memwal). + +## 0.0.1 + +### Initial Release + +- `MemWal` default client — relayer-handled embedding, SEAL encryption, Walrus upload, vector search +- `MemWalManual` manual client — client-side embedding and SEAL operations +- `withMemWal` Vercel AI SDK middleware — automatic memory recall and save +- Account management utilities — `createAccount`, `addDelegateKey`, `removeDelegateKey`, `generateDelegateKey` +- Ed25519 delegate key authentication +- Namespace-scoped memory isolation diff --git a/packages/openclaw-memory-memwal/CHANGELOG.md b/packages/openclaw-memory-memwal/CHANGELOG.md new file mode 100644 index 00000000..c42108de --- /dev/null +++ b/packages/openclaw-memory-memwal/CHANGELOG.md @@ -0,0 +1,12 @@ +# @mysten-incubation/oc-memwal + +## 0.0.1 + +### Initial Release + +- NemoClaw/OpenClaw memory plugin powered by MemWal +- Automatic memory recall via `before_prompt_build` hook +- Automatic fact capture via `agent_end` hook +- Session summary on `before_reset` hook +- CLI commands: `openclaw memwal stats`, `openclaw memwal search` +- LLM tools: `memory_search`, `memory_store` diff --git a/packages/openclaw-memory-memwal/src/capture.ts b/packages/openclaw-memory-memwal/src/capture.ts index 4749c09a..aa4dd4fd 100644 --- a/packages/openclaw-memory-memwal/src/capture.ts +++ b/packages/openclaw-memory-memwal/src/capture.ts @@ -6,12 +6,7 @@ * Based on patterns from LanceDB's shouldCapture() implementation. */ -// ============================================================================ -// Constants -// ============================================================================ - -const MIN_CAPTURE_LENGTH = 30; -const MAX_EMOJI_COUNT = 3; +import { MIN_CAPTURE_LENGTH, MAX_EMOJI_COUNT } from "./constants.js"; /** Filler patterns — exact-match trivial responses. */ const FILLER_PATTERN = /^(ok|okay|sure|thanks|thank you|thx|yes|yep|yeah|no|nope|nah|got it|hmm|hm|ah|oh|lol|haha|nice|cool|great|right|alright|fine|k|kk)\s*[.!?]*$/i; diff --git a/packages/openclaw-memory-memwal/src/constants.ts b/packages/openclaw-memory-memwal/src/constants.ts new file mode 100644 index 00000000..d8c0a3e7 --- /dev/null +++ b/packages/openclaw-memory-memwal/src/constants.ts @@ -0,0 +1,60 @@ +/** + * Shared numeric constants used across the plugin. + * + * Centralised here to avoid magic numbers scattered in business logic. + * Each constant documents its purpose and which modules consume it. + */ + +// ============================================================================ +// Capture filtering (capture.ts) +// ============================================================================ + +/** Minimum character length for a message to be considered capturable. */ +export const MIN_CAPTURE_LENGTH = 30; + +/** Messages with more emoji than this are treated as reactions, not facts. */ +export const MAX_EMOJI_COUNT = 3; + +// ============================================================================ +// Text extraction (format.ts) +// ============================================================================ + +/** Messages shorter than this (after tag stripping) are dropped as trivial. */ +export const MIN_EXTRACTED_TEXT_LENGTH = 10; + +// ============================================================================ +// Recall hook (hooks/recall.ts) +// ============================================================================ + +/** Prompts shorter than this skip the recall round-trip entirely. */ +export const MIN_PROMPT_LENGTH = 10; + +// ============================================================================ +// Store tool (tools/store.ts) +// ============================================================================ + +/** Minimum trimmed length for text submitted to memory_store. */ +export const MIN_STORE_TEXT_LENGTH = 3; + +/** Max extracted facts shown in the store confirmation preview. */ +export const MAX_FACT_PREVIEW_COUNT = 3; + +/** Max characters of raw text shown as fallback preview. */ +export const MAX_TEXT_PREVIEW_LENGTH = 100; + +// ============================================================================ +// Search tool (tools/search.ts) +// ============================================================================ + +/** Default result limit for memory_search when caller omits `limit`. */ +export const DEFAULT_SEARCH_LIMIT = 5; + +// ============================================================================ +// Retry (format.ts → withRetry) +// ============================================================================ + +/** Default number of retry attempts (1 = 2 total tries). */ +export const DEFAULT_RETRY_COUNT = 1; + +/** Default delay in ms between retry attempts. */ +export const DEFAULT_RETRY_DELAY_MS = 2000; diff --git a/packages/openclaw-memory-memwal/src/format.ts b/packages/openclaw-memory-memwal/src/format.ts index e4cb8be3..73bd25ec 100644 --- a/packages/openclaw-memory-memwal/src/format.ts +++ b/packages/openclaw-memory-memwal/src/format.ts @@ -3,6 +3,12 @@ * Shared by hooks, tools, and CLI. */ +import { + MIN_EXTRACTED_TEXT_LENGTH, + DEFAULT_RETRY_COUNT, + DEFAULT_RETRY_DELAY_MS, +} from "./constants.js"; + // ============================================================================ // Constants // ============================================================================ @@ -104,7 +110,7 @@ export function extractMessageTexts( // Strip our injected memory tags to prevent feedback loops, then drop // anything that's empty or trivially short after stripping text = stripMemoryTags(text).trim(); - if (text.length > 10) { + if (text.length > MIN_EXTRACTED_TEXT_LENGTH) { texts.push(text); } } @@ -130,8 +136,8 @@ export function toolError(message: string, err: unknown) { */ export async function withRetry( fn: () => Promise, - retries: number = 1, - delayMs: number = 2000, + retries: number = DEFAULT_RETRY_COUNT, + delayMs: number = DEFAULT_RETRY_DELAY_MS, ): Promise { try { return await fn(); diff --git a/packages/openclaw-memory-memwal/src/hooks/recall.ts b/packages/openclaw-memory-memwal/src/hooks/recall.ts index c6798c55..5b679abb 100644 --- a/packages/openclaw-memory-memwal/src/hooks/recall.ts +++ b/packages/openclaw-memory-memwal/src/hooks/recall.ts @@ -11,8 +11,7 @@ import { resolveAgent } from "../config.js"; import { looksLikeInjection } from "../capture.js"; import { formatMemoriesForPrompt } from "../format.js"; import type { PluginConfig } from "../types.js"; - -const MIN_PROMPT_LENGTH = 10; +import { MIN_PROMPT_LENGTH } from "../constants.js"; /** Register the before_prompt_build hook for auto-recall. */ export function registerRecallHook(api: any, client: MemWal, config: PluginConfig): void { diff --git a/packages/openclaw-memory-memwal/src/tools/search.ts b/packages/openclaw-memory-memwal/src/tools/search.ts index ca987165..4ffc81df 100644 --- a/packages/openclaw-memory-memwal/src/tools/search.ts +++ b/packages/openclaw-memory-memwal/src/tools/search.ts @@ -12,6 +12,7 @@ import { Type } from "@sinclair/typebox"; import { looksLikeInjection } from "../capture.js"; import { escapeForPrompt, toolError } from "../format.js"; import type { PluginConfig } from "../types.js"; +import { DEFAULT_SEARCH_LIMIT } from "../constants.js"; /** Register the memory_search agent tool. */ export function registerSearchTool(api: any, client: MemWal, config: PluginConfig): void { @@ -35,7 +36,7 @@ export function registerSearchTool(api: any, client: MemWal, config: PluginConfi ), }), async execute(_id: string, params: any) { - const { query, limit = 5, namespace } = params; + const { query, limit = DEFAULT_SEARCH_LIMIT, namespace } = params; // LLM may omit namespace (e.g. tools.allow set but hooks disabled) — fall back safely const ns = namespace || config.defaultNamespace; diff --git a/packages/openclaw-memory-memwal/src/tools/store.ts b/packages/openclaw-memory-memwal/src/tools/store.ts index 71e8f861..1fb2c4df 100644 --- a/packages/openclaw-memory-memwal/src/tools/store.ts +++ b/packages/openclaw-memory-memwal/src/tools/store.ts @@ -11,6 +11,7 @@ import { Type } from "@sinclair/typebox"; import { looksLikeInjection } from "../capture.js"; import { toolError } from "../format.js"; import type { PluginConfig } from "../types.js"; +import { MIN_STORE_TEXT_LENGTH, MAX_FACT_PREVIEW_COUNT, MAX_TEXT_PREVIEW_LENGTH } from "../constants.js"; /** Register the memory_store agent tool. */ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig): void { @@ -52,7 +53,7 @@ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig }; } - if (!text || text.trim().length < 3) { + if (!text || text.trim().length < MIN_STORE_TEXT_LENGTH) { return { content: [ { @@ -71,8 +72,8 @@ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig // Show first 3 extracted facts as confirmation, or raw text truncation as fallback const preview = result.facts ?.map((f: any) => f.text) - .slice(0, 3) - .join("; ") ?? text.slice(0, 100); + .slice(0, MAX_FACT_PREVIEW_COUNT) + .join("; ") ?? text.slice(0, MAX_TEXT_PREVIEW_LENGTH); return { content: [ diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md new file mode 100644 index 00000000..ea0c0192 --- /dev/null +++ b/packages/sdk/CHANGELOG.md @@ -0,0 +1,12 @@ +# @mysten-incubation/memwal + +## 0.0.1 + +### Initial Release + +- `MemWal` default client — relayer-handled embedding, SEAL encryption, Walrus upload, vector search +- `MemWalManual` manual client — client-side embedding and SEAL operations +- `withMemWal` Vercel AI SDK middleware — automatic memory recall and save +- Account management utilities — `createAccount`, `addDelegateKey`, `removeDelegateKey`, `generateDelegateKey` +- Ed25519 delegate key authentication +- Namespace-scoped memory isolation diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23f0458a..ae6e5569 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -702,8 +702,8 @@ importers: specifier: ^13.7.0 version: 13.12.0(react@19.0.1) '@mysten-incubation/memwal': - specifier: workspace:* - version: link:../../packages/sdk + specifier: 0.0.1 + version: 0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76) '@noble/ed25519': specifier: ^2.2.3 version: 2.3.0 @@ -973,6 +973,25 @@ importers: specifier: ^4 version: 4.2.441(@radix-ui/react-popover@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/node@24.12.0)(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(typescript@5.9.3) + packages/openclaw-memory-memwal: + dependencies: + '@mysten-incubation/memwal': + specifier: ^0.0.1 + version: 0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76) + '@sinclair/typebox': + specifier: 0.34.48 + version: 0.34.48 + zod: + specifier: ^3.23.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.19.15 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + packages/sdk: dependencies: '@mysten/seal': @@ -3045,6 +3064,26 @@ packages: resolution: {integrity: sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==} engines: {node: '>=18'} + '@mysten-incubation/memwal@0.0.1': + resolution: {integrity: sha512-kRAFFJBdk3D9XvGHZdPOrnz2x4C7dwCRf0xTaeLFAVTgVwfpk3GmOnJZ1O+pAQyrAhweAzBXNXBWutShnPWgJg==} + peerDependencies: + '@mysten/seal': '>=1.1.0' + '@mysten/sui': '>=2.5.0' + '@mysten/walrus': '>=1.0.3' + ai: '>=4.0.0' + zod: ^3.23.0 + peerDependenciesMeta: + '@mysten/seal': + optional: true + '@mysten/sui': + optional: true + '@mysten/walrus': + optional: true + ai: + optional: true + zod: + optional: true + '@mysten/bcs@1.2.0': resolution: {integrity: sha512-LuKonrGdGW7dq/EM6U2L9/as7dFwnhZnsnINzB/vu08Xfrj0qzWwpLOiXagAa5yZOPLK7anRZydMonczFkUPzA==} @@ -4458,6 +4497,9 @@ packages: '@shikijs/vscode-textmate@10.0.2': resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + '@sinclair/typebox@0.34.48': + resolution: {integrity: sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==} + '@sindresorhus/is@5.6.0': resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} engines: {node: '>=14.16'} @@ -13486,6 +13528,17 @@ snapshots: outvariant: 1.4.3 strict-event-emitter: 0.5.1 + '@mysten-incubation/memwal@0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76)': + dependencies: + '@noble/ed25519': 2.3.0 + '@noble/hashes': 2.0.1 + optionalDependencies: + '@mysten/seal': 1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)) + '@mysten/sui': 2.8.0(typescript@5.9.3) + '@mysten/walrus': 1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)) + ai: 6.0.37(zod@3.25.76) + zod: 3.25.76 + '@mysten/bcs@1.2.0': dependencies: bs58: 6.0.0 @@ -16528,6 +16581,8 @@ snapshots: '@shikijs/vscode-textmate@10.0.2': {} + '@sinclair/typebox@0.34.48': {} + '@sindresorhus/is@5.6.0': {} '@sindresorhus/merge-streams@4.0.0': {} @@ -16909,7 +16964,7 @@ snapshots: '@types/cors@2.8.19': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 '@types/d3-array@3.2.2': {} @@ -16958,7 +17013,7 @@ snapshots: '@types/es-aggregate-error@1.0.6': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 '@types/estree-jsx@1.0.5': dependencies: @@ -17088,7 +17143,7 @@ snapshots: '@types/yauzl@2.10.3': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 optional: true '@typescript-eslint/eslint-plugin@8.57.0(@typescript-eslint/parser@8.57.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3)': @@ -18646,7 +18701,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.19 - '@types/node': 20.19.37 + '@types/node': 22.19.15 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -22163,7 +22218,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.19.37 + '@types/node': 22.19.15 long: 5.3.2 proxy-addr@2.0.7: diff --git a/services/server/Cargo.lock b/services/server/Cargo.lock index 8ef8da12..980c8c2b 100644 --- a/services/server/Cargo.lock +++ b/services/server/Cargo.lock @@ -32,6 +32,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -205,6 +214,20 @@ dependencies = [ "windows-link", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -842,7 +865,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "system-configuration", "tokio", "tower-service", @@ -1010,6 +1033,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1145,6 +1177,7 @@ dependencies = [ "futures", "hex", "pgvector", + "redis", "reqwest", "serde", "serde_json", @@ -1212,6 +1245,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -1488,6 +1531,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "redis" +version = "0.27.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "combine", + "futures-util", + "itertools", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.5.10", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1787,6 +1854,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1848,6 +1921,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -2247,7 +2330,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] diff --git a/services/server/Cargo.toml b/services/server/Cargo.toml index 44c69541..004b6fd4 100644 --- a/services/server/Cargo.toml +++ b/services/server/Cargo.toml @@ -42,6 +42,9 @@ walrus_rs = "0.1" # Async utilities futures = "0.3" +# Rate limiting (Redis-backed) +redis = { version = "0.27", features = ["tokio-comp"] } + # Utils uuid = { version = "1", features = ["v4"] } chrono = "0.4" diff --git a/services/server/migrations/003_rate_limiter.sql b/services/server/migrations/003_rate_limiter.sql new file mode 100644 index 00000000..12cb101f --- /dev/null +++ b/services/server/migrations/003_rate_limiter.sql @@ -0,0 +1,13 @@ +-- memwal — Storage Quota Tracking +-- Rate limiting is handled by Redis (no PostgreSQL table needed). +-- Storage quota is tracked per-row in vector_entries. + +-- ============================================================ +-- Storage quota: track blob size in vector_entries +-- ============================================================ +-- blob_size_bytes tracks the size of each encrypted blob uploaded. +-- Total storage per user = SUM(blob_size_bytes) WHERE owner = $1. +-- When blobs expire and are cleaned up (delete_by_blob_id), quota +-- is automatically reduced. +ALTER TABLE vector_entries + ADD COLUMN IF NOT EXISTS blob_size_bytes BIGINT NOT NULL DEFAULT 0; diff --git a/services/server/scripts/sidecar-server.ts b/services/server/scripts/sidecar-server.ts index 19220ef1..be3f70ef 100644 --- a/services/server/scripts/sidecar-server.ts +++ b/services/server/scripts/sidecar-server.ts @@ -51,7 +51,7 @@ const WALRUS_UPLOAD_RELAY_URL = process.env.WALRUS_UPLOAD_RELAY_URL || ( : "https://upload-relay.mainnet.walrus.space" ); -const DEFAULT_WALRUS_EPOCHS = SUI_NETWORK === "testnet" ? 50 : 2; +const DEFAULT_WALRUS_EPOCHS = SUI_NETWORK === "testnet" ? 50 : 3; const suiClient = new SuiJsonRpcClient({ url: getJsonRpcFullnodeUrl(SUI_NETWORK), @@ -76,17 +76,96 @@ const walrusClient = new WalrusClient({ }, }); +const COIN_WITH_BALANCE_INTENT = "CoinWithBalance"; +const GAS_INTENT_TYPE = "gas"; +const SUI_TYPE = "0x2::sui::SUI"; +type TxIntentCommand = { + $kind?: string; + $Intent?: { + name?: string; + data?: { type?: string }; + }; +}; +type TxDataWithCommands = { commands: TxIntentCommand[] }; +type UploadRelayTipConfigResponse = { + send_tip?: { + address?: string; + }; +}; + +/** + * Rewrite CoinWithBalance "gas" intents to explicit SUI coin type so Enoki + * sponsorship can build the transaction (Enoki rejects GasCoin tx arguments). + */ +function patchGasCoinIntents(tx: Transaction): void { + tx.addSerializationPlugin(async (transactionData: TxDataWithCommands, _buildOptions, next) => { + let patched = 0; + for (const command of transactionData.commands) { + if ( + command.$kind === "$Intent" && + command.$Intent?.name === COIN_WITH_BALANCE_INTENT && + command.$Intent?.data?.type === GAS_INTENT_TYPE + ) { + command.$Intent.data.type = SUI_TYPE; + patched += 1; + } + } + + if (patched > 0) { + console.log(`[patch] converted ${patched} CoinWithBalance intent(s) from GasCoin -> sender SUI coins`); + } + + await next(); + }); +} + const ENOKI_API_BASE_URL = "https://api.enoki.mystenlabs.com/v1"; const enokiApiKey = process.env.ENOKI_API_KEY; const enokiNetwork = (process.env.ENOKI_NETWORK || process.env.SUI_NETWORK || "mainnet") as | "mainnet" | "testnet" | "devnet"; +const ENOKI_FALLBACK_TO_DIRECT_SIGN = (() => { + const raw = (process.env.ENOKI_FALLBACK_TO_DIRECT_SIGN || "true").trim().toLowerCase(); + return raw !== "0" && raw !== "false" && raw !== "no"; +})(); type EnokiDataWrapper = { data: T }; type EnokiSponsorResponse = { bytes: string; digest: string }; type EnokiExecuteResponse = { digest: string }; const signerUploadQueues = new Map>(); +let uploadRelayTipAddressCache: string | null | undefined = undefined; + +function dedupeAddresses(addresses: (string | null | undefined)[]): string[] { + return [...new Set(addresses.filter((addr): addr is string => typeof addr === "string" && addr.length > 0))]; +} + +async function getUploadRelayTipAddress(): Promise { + if (uploadRelayTipAddressCache !== undefined) { + return uploadRelayTipAddressCache; + } + + try { + const resp = await fetch(`${WALRUS_UPLOAD_RELAY_URL}/v1/tip-config`); + if (!resp.ok) { + throw new Error(`tip-config request failed (${resp.status})`); + } + + const json = await resp.json() as UploadRelayTipConfigResponse; + const address = json.send_tip?.address; + if (typeof address === "string" && address.startsWith("0x")) { + uploadRelayTipAddressCache = address; + return address; + } + + uploadRelayTipAddressCache = null; + return null; + } catch (err: any) { + console.warn(`[upload-relay] could not load tip-config: ${err.message || err}`); + // Don't cache transient failures; retry on next request. + return null; + } +} async function callEnoki(path: string, payload: unknown): Promise { if (!enokiApiKey) { @@ -147,7 +226,13 @@ async function executeWithEnokiSponsor(tx: Transaction, signer: Ed25519Keypair, return executed.digest; } catch (err: any) { - console.warn(`[enoki-sponsor] sponsor failed, falling back to direct signing: ${err.message}`); + const errMsg = err?.message || String(err); + if (!ENOKI_FALLBACK_TO_DIRECT_SIGN) { + console.error(`[enoki-sponsor] sponsor failed and fallback disabled: ${errMsg}`); + throw err; + } + + console.warn(`[enoki-sponsor] sponsor failed, falling back to direct signing: ${errMsg}`); const direct = await suiClient.signAndExecuteTransaction({ signer, transaction: tx, @@ -454,12 +539,16 @@ app.post("/walrus/upload", async (req, res) => { }, }); - // Wait until register tx is confirmed before starting upload/certify. - // NOTE: Walrus register uses GasCoin internally — cannot be Enoki-sponsored - const registerResult = await suiClient.signAndExecuteTransaction({ signer, transaction: registerTx }); - await suiClient.waitForTransaction({ digest: registerResult.digest }); + // Patch: convert GasCoin intents → sender's SUI coins. + // Enoki rejects GasCoin as tx argument, but relay requires the tip. + // After patching, signer pays tip from own SUI; Enoki sponsors gas. + patchGasCoinIntents(registerTx); + const tipRecipient = await getUploadRelayTipAddress(); + const registerAllowedAddresses = dedupeAddresses([signerAddress, tipRecipient]); + const registerDigest = await executeWithEnokiSponsor(registerTx, signer, registerAllowedAddresses); + await suiClient.waitForTransaction({ digest: registerDigest }); - await flow.upload({ digest: registerResult.digest }); + await flow.upload({ digest: registerDigest }); const certifyTx = flow.certify(); // Wait until certify tx is confirmed before returning this upload. @@ -525,9 +614,9 @@ app.post("/walrus/upload", async (req, res) => { // Transfer blob to user metaTx.transferObjects([blobArg], owner); - const metaDigest = await executeWithEnokiSponsor(metaTx, signer, [owner]); + const metaDigest = await executeWithEnokiSponsor(metaTx, signer, dedupeAddresses([signerAddress, owner])); await suiClient.waitForTransaction({ digest: metaDigest }); - console.error(`[walrus/upload] metadata set + transferred blob ${blobObjectId} to ${owner} (ns=${namespace})`); + console.log(`[walrus/upload] metadata set + transferred blob ${blobObjectId} to ${owner} (ns=${namespace})`); } catch (metaErr: any) { // Non-fatal: blob is uploaded but metadata/transfer failed console.error(`[walrus/upload] metadata+transfer failed: ${metaErr.message}`); diff --git a/services/server/src/db.rs b/services/server/src/db.rs index 97d2e7fc..02688217 100644 --- a/services/server/src/db.rs +++ b/services/server/src/db.rs @@ -30,12 +30,18 @@ impl VectorDb { .await .map_err(|e| AppError::Internal(format!("Failed to run migration 002: {}", e)))?; + let migration_003 = include_str!("../migrations/003_rate_limiter.sql"); + sqlx::raw_sql(migration_003) + .execute(&pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to run migration 003: {}", e)))?; + tracing::info!("database connected and migrations applied"); Ok(Self { pool }) } - /// Insert a vector entry + /// Insert a vector entry (with blob size tracking for storage quota) pub async fn insert_vector( &self, id: &str, @@ -43,23 +49,25 @@ impl VectorDb { namespace: &str, blob_id: &str, vector: &[f32], + blob_size_bytes: i64, ) -> Result<(), AppError> { let embedding = Vector::from(vector.to_vec()); sqlx::query( - "INSERT INTO vector_entries (id, owner, namespace, blob_id, embedding) - VALUES ($1, $2, $3, $4, $5)", + "INSERT INTO vector_entries (id, owner, namespace, blob_id, embedding, blob_size_bytes) + VALUES ($1, $2, $3, $4, $5, $6)", ) .bind(id) .bind(owner) .bind(namespace) .bind(blob_id) .bind(embedding) + .bind(blob_size_bytes) .execute(&self.pool) .await .map_err(|e| AppError::Internal(format!("Failed to insert vector: {}", e)))?; - tracing::debug!("inserted vector: id={}, blob_id={}, owner={}, ns={}", id, blob_id, owner, namespace); + tracing::debug!("inserted vector: id={}, blob_id={}, owner={}, ns={}, size={}B", id, blob_id, owner, namespace, blob_size_bytes); Ok(()) } @@ -198,6 +206,23 @@ impl VectorDb { Ok(()) } + // ============================================================ + // Storage Quota (still PostgreSQL — tracks per-row blob sizes) + // ============================================================ + + /// Get total storage used by a user (sum of blob_size_bytes for active entries). + pub async fn get_storage_used(&self, owner: &str) -> Result { + let row: (i64,) = sqlx::query_as( + "SELECT COALESCE(SUM(blob_size_bytes)::BIGINT, 0) FROM vector_entries WHERE owner = $1", + ) + .bind(owner) + .fetch_one(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to get storage used: {}", e)))?; + + Ok(row.0) + } + // ============================================================ // Accounts (populated by v2-indexer) // ============================================================ diff --git a/services/server/src/main.rs b/services/server/src/main.rs index e90669d5..f64276dd 100644 --- a/services/server/src/main.rs +++ b/services/server/src/main.rs @@ -1,5 +1,6 @@ mod auth; mod db; +mod rate_limit; mod routes; mod seal; mod sui; @@ -34,6 +35,12 @@ async fn main() { tracing::info!(" package id: {}", config.package_id); tracing::info!(" registry id: {}", config.registry_id); tracing::info!(" memwal account: {}", config.memwal_account_id.as_deref().unwrap_or("(from client header)")); + tracing::info!(" rate limit: burst={}/min, sustained={}/hr, per-key={}/min, quota={}MB/user", + config.rate_limit.max_requests_per_minute, + config.rate_limit.max_requests_per_hour, + config.rate_limit.max_requests_per_delegate_key, + config.rate_limit.max_storage_bytes / 1_048_576 + ); // Start TS sidecar HTTP server (SEAL + Walrus operations) let sidecar_url = config.sidecar_url.clone(); @@ -98,6 +105,12 @@ async fn main() { // Build key pool for parallel Walrus uploads let key_pool = KeyPool::new(config.sui_private_keys.clone()); + // Initialize Redis for rate limiting + let redis = rate_limit::create_redis_client(&config.rate_limit.redis_url) + .await + .expect("Failed to connect to Redis for rate limiting"); + tracing::info!(" Redis: connected at {}", config.rate_limit.redis_url); + // Shared application state let state = Arc::new(AppState { db, @@ -105,6 +118,7 @@ async fn main() { http_client, walrus_client, key_pool, + redis, }); // Build routes @@ -118,6 +132,12 @@ async fn main() { .route("/api/analyze", post(routes::analyze)) .route("/api/ask", post(routes::ask)) .route("/api/restore", post(routes::restore)) + // Router::layer runs middleware bottom-to-top (last added runs first). + // Keep auth outer so AuthInfo is in request extensions before rate limiting reads it. + .layer(middleware::from_fn_with_state( + state.clone(), + rate_limit::rate_limit_middleware, + )) .layer(middleware::from_fn_with_state( state.clone(), auth::verify_signature, diff --git a/services/server/src/rate_limit.rs b/services/server/src/rate_limit.rs new file mode 100644 index 00000000..e9dc490b --- /dev/null +++ b/services/server/src/rate_limit.rs @@ -0,0 +1,326 @@ +use axum::{ + extract::{Request, State}, + http::StatusCode, + middleware::Next, + response::Response, +}; +use std::sync::Arc; + +use crate::types::{AppError, AppState}; + +// ============================================================ +// Rate Limit Configuration +// ============================================================ + +#[derive(Debug, Clone)] +pub struct RateLimitConfig { + // --- Per-account burst window --- + /// Maximum weighted requests per minute per user (default: 60) + pub max_requests_per_minute: i64, + + // --- Per-account sustained window --- + /// Maximum weighted requests per hour per user (default: 500) + pub max_requests_per_hour: i64, + + // --- Per-delegate-key window --- + /// Maximum weighted requests per minute per delegate key (default: 30) + pub max_requests_per_delegate_key: i64, + + // --- Storage quota --- + /// Maximum storage per user in bytes (default: 1 GB) + pub max_storage_bytes: i64, + + /// Redis URL (default: redis://localhost:6379) + pub redis_url: String, +} + +impl Default for RateLimitConfig { + fn default() -> Self { + Self { + max_requests_per_minute: 60, + max_requests_per_hour: 500, + max_requests_per_delegate_key: 30, + max_storage_bytes: 1_073_741_824, // 1 GB + redis_url: "redis://localhost:6379".to_string(), + } + } +} + +impl RateLimitConfig { + pub fn from_env() -> Self { + let mut config = Self::default(); + + if let Ok(val) = std::env::var("RATE_LIMIT_REQUESTS_PER_MINUTE") { + if let Ok(n) = val.parse::() { + config.max_requests_per_minute = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_REQUESTS_PER_HOUR") { + if let Ok(n) = val.parse::() { + config.max_requests_per_hour = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_DELEGATE_KEY_PER_MINUTE") { + if let Ok(n) = val.parse::() { + config.max_requests_per_delegate_key = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_STORAGE_BYTES") { + if let Ok(n) = val.parse::() { + config.max_storage_bytes = n; + } + } + + if let Ok(val) = std::env::var("REDIS_URL") { + config.redis_url = val; + } + + config + } +} + +// ============================================================ +// Cost Weights — per endpoint +// ============================================================ + +/// Get the cost weight for a given API path. +/// +/// Expensive endpoints (embedding + encrypt + Walrus upload + LLM) +/// consume more of the rate limit budget than cheap read endpoints. +fn endpoint_weight(path: &str) -> i64 { + match path { + "/api/analyze" => 10, // LLM extract + N × (embed + encrypt + upload) + "/api/remember" => 5, // embed + SEAL encrypt + Walrus upload + "/api/remember/manual" => 3, // Walrus upload only (client did embed/encrypt) + "/api/restore" => 3, // download + decrypt + re-embed + "/api/ask" => 2, // recall + LLM + _ => 1, // recall, recall/manual, etc. + } +} + +// ============================================================ +// Redis Client +// ============================================================ + +/// Create a Redis multiplexed connection for shared use across the app. +pub async fn create_redis_client(redis_url: &str) -> Result { + let client = redis::Client::open(redis_url) + .map_err(|e| format!("Failed to create Redis client: {}", e))?; + + let conn = client.get_multiplexed_async_connection() + .await + .map_err(|e| format!("Failed to connect to Redis: {}", e))?; + + Ok(conn) +} + +// ============================================================ +// Sliding Window Helpers +// ============================================================ + +/// Check the current count in a Redis sorted set sliding window. +/// Returns the count of entries within the window. +async fn check_window( + redis: &mut redis::aio::MultiplexedConnection, + key: &str, + window_start: f64, +) -> Result { + let result: ((), i64) = redis::pipe() + .atomic() + .zrembyscore(key, 0.0_f64, window_start) + .zcard(key) + .query_async(redis) + .await?; + + Ok(result.1) +} + +/// Record weighted entries in a Redis sorted set sliding window. +/// Adds `weight` entries and sets TTL. +async fn record_in_window( + redis: &mut redis::aio::MultiplexedConnection, + key: &str, + now: f64, + weight: i64, + ttl_seconds: i64, +) { + let mut pipe = redis::pipe(); + for i in 0..weight { + // Use fractional offsets to create unique members + let ts = now + i as f64 * 0.001; + pipe.zadd(key, ts, format!("{}", ts)); + } + pipe.expire(key, ttl_seconds); + + let _: Result<(), _> = pipe.query_async(redis).await; +} + +// ============================================================ +// Rate Limit Response +// ============================================================ + +/// Build a 429 response with JSON body and Retry-After header. +fn rate_limit_response(layer: &str, limit: i64, window: &str, retry_after: u64) -> Response { + let body = serde_json::json!({ + "error": "Rate limit exceeded", + "layer": layer, + "limit": format!("{} weighted-requests/{}", limit, window), + "retry_after_seconds": retry_after, + }); + + axum::response::Response::builder() + .status(StatusCode::TOO_MANY_REQUESTS) + .header("Content-Type", "application/json") + .header("Retry-After", retry_after.to_string()) + .body(axum::body::Body::from(serde_json::to_string(&body).unwrap())) + .unwrap() +} + +// ============================================================ +// Rate Limit Middleware +// ============================================================ + +/// Multi-layer rate limiting middleware for authenticated routes. +/// +/// Checks 3 layers (all must pass): +/// 1. Per-delegate-key: 30 weighted-req/min (prevents compromised key abuse) +/// 2. Per-account burst: 60 weighted-req/min (prevents spam) +/// 3. Per-account sustained: 500 weighted-req/hour (prevents slow-burn) +/// +/// Endpoints are cost-weighted: +/// analyze=10, remember=5, remember/manual=3, restore=3, ask=2, recall=1 +/// +/// Returns 429 Too Many Requests with JSON body if any layer exceeds its limit. +pub async fn rate_limit_middleware( + State(state): State>, + request: Request, + next: Next, +) -> Response { + // Extract auth info (set by auth middleware) + let auth_info = request + .extensions() + .get::() + .cloned(); + + let auth = match auth_info { + Some(a) => a, + None => { + // No auth info = not an authenticated route, skip rate limiting + return next.run(request).await; + } + }; + + let config = &state.config.rate_limit; + let mut redis = state.redis.clone(); + let now = chrono::Utc::now().timestamp_millis() as f64; + + // Determine cost weight based on endpoint + let weight = endpoint_weight(request.uri().path()); + + // --- Layer 1: Per-delegate-key (burst) --- + let dk_key = format!("rate:dk:{}", auth.public_key); + let dk_window_start = now - 60_000.0; // 1 min window + + match check_window(&mut redis, &dk_key, dk_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_delegate_key { + tracing::warn!( + "rate limit [delegate-key]: key={}... count={}/{} weight={} path={}", + &auth.public_key[..16], count, + config.max_requests_per_delegate_key, weight, request.uri().path() + ); + return rate_limit_response("delegate_key", config.max_requests_per_delegate_key, "min", 60); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (dk): {}, allowing", e); + } + } + + // --- Layer 2: Per-account burst (1 min) --- + let burst_key = format!("rate:{}", auth.owner); + let burst_window_start = now - 60_000.0; + + match check_window(&mut redis, &burst_key, burst_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_minute { + tracing::warn!( + "rate limit [burst]: owner={} count={}/{} weight={} path={}", + auth.owner, count, config.max_requests_per_minute, weight, request.uri().path() + ); + return rate_limit_response("account_burst", config.max_requests_per_minute, "min", 60); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (burst): {}, allowing", e); + } + } + + // --- Layer 3: Per-account sustained (1 hour) --- + let hourly_key = format!("rate:hr:{}", auth.owner); + let hourly_window_start = now - 3_600_000.0; + + match check_window(&mut redis, &hourly_key, hourly_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_hour { + tracing::warn!( + "rate limit [sustained]: owner={} count={}/{} weight={} path={}", + auth.owner, count, config.max_requests_per_hour, weight, request.uri().path() + ); + return rate_limit_response("account_sustained", config.max_requests_per_hour, "hour", 300); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (sustained): {}, allowing", e); + } + } + + // --- All checks passed: record weighted entries in all 3 windows --- + record_in_window(&mut redis, &dk_key, now, weight, 120).await; // TTL 2min + record_in_window(&mut redis, &burst_key, now + 0.1, weight, 120).await; // offset to avoid collision + record_in_window(&mut redis, &hourly_key, now + 0.2, weight, 3700).await; // TTL ~1hr+buffer + + next.run(request).await +} + +// ============================================================ +// Storage Quota Check (called from routes, not middleware) +// ============================================================ + +/// Check if a user has enough storage quota for a new blob. +/// +/// Storage tracking still uses PostgreSQL (it's per-row in vector_entries). +/// Returns `Ok(())` if within quota, `Err(AppError::QuotaExceeded)` if not. +pub async fn check_storage_quota( + state: &AppState, + owner: &str, + additional_bytes: i64, +) -> Result<(), AppError> { + let max_bytes = state.config.rate_limit.max_storage_bytes; + + // 0 or negative means unlimited + if max_bytes <= 0 { + return Ok(()); + } + + let used = state.db.get_storage_used(owner).await?; + let projected = used + additional_bytes; + + if projected > max_bytes { + let used_mb = used as f64 / 1_048_576.0; + let max_mb = max_bytes as f64 / 1_048_576.0; + tracing::warn!( + "storage quota exceeded: owner={} used={:.1}MB + {:.1}MB > max={:.1}MB", + owner, used_mb, additional_bytes as f64 / 1_048_576.0, max_mb + ); + return Err(AppError::QuotaExceeded(format!( + "Storage quota exceeded: {:.1}MB used of {:.1}MB allowed", + used_mb, max_mb + ))); + } + + Ok(()) +} diff --git a/services/server/src/routes.rs b/services/server/src/routes.rs index ccaf5314..df8396cc 100644 --- a/services/server/src/routes.rs +++ b/services/server/src/routes.rs @@ -6,9 +6,23 @@ use std::sync::Arc; use crate::seal; use crate::walrus; +use crate::rate_limit; use crate::types::*; use crate::db::VectorDb; +/// Truncate a string to at most `max_bytes` bytes without splitting a UTF-8 +/// character. Falls back to the nearest char boundary when `max_bytes` lands +/// inside a multi-byte sequence (e.g. emoji). +fn truncate_str(s: &str, max_bytes: usize) -> &str { + if s.len() <= max_bytes { + return s; + } + let mut end = max_bytes; + while !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] +} // ============================================================ // Embedding — OpenRouter/OpenAI API (with mock fallback) @@ -119,7 +133,11 @@ pub async fn remember( let owner = &auth.owner; let text = &body.text; let namespace = &body.namespace; - tracing::info!("remember: text=\"{}...\" owner={} ns={}", &text[..text.len().min(50)], owner, namespace); + tracing::info!("remember: text=\"{}...\" owner={} ns={}", truncate_str(text, 50), owner, namespace); + + // Check storage quota before processing + let text_bytes = text.as_bytes().len() as i64; + rate_limit::check_storage_quota(&state, owner, text_bytes).await?; // Step 1: Embed text + SEAL encrypt concurrently (they're independent) let embed_fut = generate_embedding(&state.http_client, &state.config, text); @@ -132,18 +150,19 @@ pub async fn remember( let encrypted = encrypted_result?; // Step 2: Upload encrypted blob → Walrus (via sidecar) - let sui_key = state.config.sui_private_key.as_deref().ok_or_else(|| { - AppError::Internal("SERVER_SUI_PRIVATE_KEY required for Walrus upload".into()) - })?; + let sui_key = state.key_pool.next() + .map(|s| s.to_string()) + .ok_or_else(|| AppError::Internal("No Sui keys configured (set SERVER_SUI_PRIVATE_KEYS or SERVER_SUI_PRIVATE_KEY)".into()))?; let upload_result = walrus::upload_blob( &state.http_client, &state.config.sidecar_url, - &encrypted, 50, owner, sui_key, namespace, &state.config.package_id, + &encrypted, 50, owner, &sui_key, namespace, &state.config.package_id, ).await?; let blob_id = upload_result.blob_id; // Step 3: Store {vector, blobId, namespace} in Vector DB + let blob_size = encrypted.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, &blob_id, &vector).await?; + state.db.insert_vector(&id, owner, namespace, &blob_id, &vector, blob_size).await?; tracing::info!( "remember complete: blob_id={}, owner={}, ns={}, dims={}", @@ -178,7 +197,7 @@ pub async fn recall( // Owner is derived from delegate key via onchain verification (auth middleware) let owner = &auth.owner; let namespace = &body.namespace; - tracing::info!("recall: query=\"{}...\" owner={} ns={}", &body.query[..body.query.len().min(50)], owner, namespace); + tracing::info!("recall: query=\"{}...\" owner={} ns={}", truncate_str(&body.query, 50), owner, namespace); // Use delegate key from SDK for SEAL decryption (falls back to server key) let private_key = auth.delegate_key.as_deref() @@ -231,7 +250,15 @@ pub async fn recall( } } Err(e) => { - tracing::warn!("Failed to SEAL decrypt blob {}: {}", blob_id, e); + let err_str = e.to_string(); + let is_permanent = err_str.contains("Not enough shares") + || err_str.contains("decrypt failed"); + if is_permanent { + tracing::warn!("SEAL decrypt permanently failed for blob {}, cleaning up: {}", blob_id, e); + cleanup_expired_blob(db, &blob_id).await; + } else { + tracing::warn!("Failed to SEAL decrypt blob {}: {}", blob_id, e); + } None } } @@ -283,10 +310,13 @@ pub async fn remember_manual( .decode(&body.encrypted_data) .map_err(|e| AppError::BadRequest(format!("encrypted_data is not valid base64: {}", e)))?; - // Upload encrypted bytes to Walrus via sidecar (server pays gas) - let sui_key = state.config.sui_private_key.as_deref().ok_or_else(|| { - AppError::Internal("SERVER_SUI_PRIVATE_KEY not configured for Walrus upload".into()) - })?; + // Check storage quota before upload + rate_limit::check_storage_quota(&state, owner, encrypted_bytes.len() as i64).await?; + + // Upload encrypted bytes to Walrus via sidecar (pool key pays gas) + let sui_key = state.key_pool.next() + .map(|s| s.to_string()) + .ok_or_else(|| AppError::Internal("No Sui keys configured (set SERVER_SUI_PRIVATE_KEYS or SERVER_SUI_PRIVATE_KEY)".into()))?; let upload = walrus::upload_blob( &state.http_client, @@ -294,7 +324,7 @@ pub async fn remember_manual( &encrypted_bytes, 50, owner, - sui_key, + &sui_key, namespace, &state.config.package_id, ) @@ -304,8 +334,9 @@ pub async fn remember_manual( tracing::info!("remember_manual: walrus upload ok blob_id={}", blob_id); // Store {vector, blobId, namespace} in Vector DB + let blob_size = encrypted_bytes.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, &blob_id, &body.vector).await?; + state.db.insert_vector(&id, owner, namespace, &blob_id, &body.vector, blob_size).await?; tracing::info!("remember_manual complete: id={}, blob_id={}, ns={}", id, blob_id, namespace); @@ -368,7 +399,7 @@ pub async fn analyze( let owner = &auth.owner; let namespace = &body.namespace; - tracing::info!("analyze: text=\"{}...\" owner={} ns={}", &body.text[..body.text.len().min(50)], owner, namespace); + tracing::info!("analyze: text=\"{}...\" owner={} ns={}", truncate_str(&body.text, 50), owner, namespace); // Step 1: Extract facts using LLM let facts = extract_facts_llm(&state.http_client, &state.config, &body.text).await?; @@ -382,6 +413,10 @@ pub async fn analyze( })); } + // Check storage quota before processing all facts + let total_text_bytes: i64 = facts.iter().map(|f| f.as_bytes().len() as i64).sum(); + rate_limit::check_storage_quota(&state, owner, total_text_bytes).await?; + // Step 2: Process all facts concurrently (embed + encrypt → upload → store) // Each fact gets its own key from the pool so sidecar can upload them in parallel // (different signer addresses bypass the per-signer serialization lock). @@ -414,8 +449,9 @@ pub async fn analyze( ).await?; // Store in Vector DB with namespace + let blob_size = encrypted.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, &owner, &namespace, &upload_result.blob_id, &vector).await?; + state.db.insert_vector(&id, &owner, &namespace, &upload_result.blob_id, &vector, blob_size).await?; Ok::(AnalyzedFact { text: fact_text, @@ -590,7 +626,7 @@ pub async fn ask( let owner = &auth.owner; let namespace = &body.namespace; let limit = body.limit.unwrap_or(5); - tracing::info!("ask: question=\"{}...\" owner={} ns={}", &body.question[..body.question.len().min(50)], owner, namespace); + tracing::info!("ask: question=\"{}...\" owner={} ns={}", truncate_str(&body.question, 50), owner, namespace); // Step 1: Recall relevant memories let query_vector = generate_embedding(&state.http_client, &state.config, &body.question).await?; @@ -855,6 +891,12 @@ pub async fn restore( .flatten() .collect(); + // Preserve encrypted blob sizes so restored rows still contribute to storage quota. + let blob_sizes: std::collections::HashMap = downloaded + .iter() + .map(|(blob_id, data)| (blob_id.clone(), data.len() as i64)) + .collect(); + if downloaded.is_empty() { return Ok(Json(RestoreResponse { restored: 0, @@ -935,7 +977,16 @@ pub async fn restore( let restored = results.len(); for (blob_id, vector) in &results { let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, blob_id, vector).await?; + let blob_size = blob_sizes.get(blob_id).copied().unwrap_or_else(|| { + tracing::warn!( + "restore: missing blob size for {}, defaulting to 0 for quota tracking", + blob_id + ); + 0 + }); + state.db + .insert_vector(&id, owner, namespace, blob_id, vector, blob_size) + .await?; } tracing::info!( diff --git a/services/server/src/types.rs b/services/server/src/types.rs index 8125e95d..ad82cb8f 100644 --- a/services/server/src/types.rs +++ b/services/server/src/types.rs @@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use serde::{Deserialize, Serialize}; use crate::db::VectorDb; +use crate::rate_limit::RateLimitConfig; // ============================================================ // App State (shared across routes + middleware) @@ -15,6 +16,8 @@ pub struct AppState { pub walrus_client: walrus_rs::WalrusClient, /// Round-robin pool of Sui private keys for parallel Walrus uploads pub key_pool: KeyPool, + /// Redis multiplexed connection for rate limiting + pub redis: redis::aio::MultiplexedConnection, } // ============================================================ @@ -75,6 +78,8 @@ pub struct Config { pub registry_id: String, /// URL of the SEAL/Walrus TS sidecar HTTP server pub sidecar_url: String, + /// Rate limiting configuration + pub rate_limit: RateLimitConfig, } impl Config { @@ -124,6 +129,7 @@ impl Config { .expect("MEMWAL_REGISTRY_ID must be set"), sidecar_url: std::env::var("SIDECAR_URL") .unwrap_or_else(|_| "http://localhost:9000".to_string()), + rate_limit: RateLimitConfig::from_env(), } } } @@ -332,6 +338,11 @@ pub enum AppError { Internal(String), /// Walrus blob not found (expired or deleted) — triggers cleanup BlobNotFound(String), + /// Rate limit exceeded (HTTP 429) + #[allow(dead_code)] + RateLimited(String), + /// Storage quota exceeded (HTTP 402) + QuotaExceeded(String), } impl std::fmt::Display for AppError { @@ -341,6 +352,8 @@ impl std::fmt::Display for AppError { AppError::Unauthorized(msg) => write!(f, "Unauthorized: {}", msg), AppError::Internal(msg) => write!(f, "Internal Error: {}", msg), AppError::BlobNotFound(msg) => write!(f, "Blob Not Found: {}", msg), + AppError::RateLimited(msg) => write!(f, "Rate Limited: {}", msg), + AppError::QuotaExceeded(msg) => write!(f, "Quota Exceeded: {}", msg), } } } @@ -355,6 +368,8 @@ impl axum::response::IntoResponse for AppError { msg.clone(), ), AppError::BlobNotFound(msg) => (axum::http::StatusCode::NOT_FOUND, msg.clone()), + AppError::RateLimited(msg) => (axum::http::StatusCode::TOO_MANY_REQUESTS, msg.clone()), + AppError::QuotaExceeded(msg) => (axum::http::StatusCode::PAYMENT_REQUIRED, msg.clone()), }; let body = serde_json::json!({ "error": message });