From 5d2538a7fce27c66ce18a912d14cb8fe990a43ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=E1=BA=A7n=20Ph=E1=BA=A1m=20Minh=20H=C3=B9ng?= <46132442+hungtranphamminh@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:24:06 +0700 Subject: [PATCH 01/10] Build: Prepare OpenClaw memory plugin for npm publish (#58) * build(openclaw-plugin): prepare package for npm publish Rename to @mysten-incubation/oc-memwal, add tsconfig, build scripts, exports, types, and engine requirements. Compiled JS output in dist/. * docs(openclaw-plugin): update install flow for npm, remove TEE references Replace manual symlink install with openclaw plugins install command. Update config examples to use oc-memwal. Remove TEE from relayer labels. * ci(openclaw-plugin): add release workflow for npm publish Auto-publish on push to main/staging/dev when plugin code changes. Uses OIDC provenance, auto-increments dev/rc versions per branch. Add typecheck script. Reset base version to 0.0.1. * fix(openclaw-plugin): update outdated links in manifest and README Replace app.memwal.com with memwal.ai in plugin uiHints. Fix relative Mintlify links to use full docs.memwal.ai URLs. --- .github/workflows/release-oc-memwal.yml | 103 ++++++++++++++++++ docs/openclaw/how-it-works.md | 4 +- docs/openclaw/quick-start.md | 47 ++------ docs/openclaw/reference.md | 4 +- packages/openclaw-memory-memwal/.gitignore | 1 + packages/openclaw-memory-memwal/README.md | 19 ++-- .../openclaw.plugin.json | 4 +- packages/openclaw-memory-memwal/package.json | 35 +++++- packages/openclaw-memory-memwal/tsconfig.json | 22 ++++ 9 files changed, 178 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/release-oc-memwal.yml create mode 100644 packages/openclaw-memory-memwal/.gitignore create mode 100644 packages/openclaw-memory-memwal/tsconfig.json diff --git a/.github/workflows/release-oc-memwal.yml b/.github/workflows/release-oc-memwal.yml new file mode 100644 index 00000000..9994accc --- /dev/null +++ b/.github/workflows/release-oc-memwal.yml @@ -0,0 +1,103 @@ +name: Release OC-MemWal Plugin + +on: + push: + branches: + - main + - staging + - dev + paths: + - 'packages/openclaw-memory-memwal/**' + - '.github/workflows/release-oc-memwal.yml' + workflow_dispatch: + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + release: + name: Version & Publish + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup PNPM + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: pnpm + registry-url: 'https://registry.npmjs.org' + + - name: Upgrade npm for OIDC Trusted Publishing + run: npm install -g npm@latest + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Build SDK (dependency) + run: pnpm build:sdk + + - name: Typecheck + run: cd packages/openclaw-memory-memwal && npm run typecheck + + - name: Build plugin + run: cd packages/openclaw-memory-memwal && npm run build + + # ── main branch → stable release (latest) ── + - name: Publish stable release + if: github.ref == 'refs/heads/main' + run: | + BASE_VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") + npm view @mysten-incubation/oc-memwal@$BASE_VERSION version 2>/dev/null \ + && echo "Version $BASE_VERSION already published, skipping" && exit 0 + cd packages/openclaw-memory-memwal && npm publish --provenance --access public + + # ── staging branch → release candidate (rc tag, auto-increment) ── + - name: Publish staging release candidate + if: github.ref == 'refs/heads/staging' + run: | + BASE_VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") + LATEST=$(npm view @mysten-incubation/oc-memwal versions --json 2>/dev/null \ + | node -p " + const versions = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); + const nums = (Array.isArray(versions) ? versions : [versions]) + .filter(v => v.startsWith('${BASE_VERSION}-rc.')) + .map(v => +v.split('-rc.')[1]) + .sort((a,b) => b - a); + nums.length ? nums[0] : -1; + " || echo "-1") + NEXT=$((LATEST + 1)) + NEW_VERSION="${BASE_VERSION}-rc.${NEXT}" + echo "Publishing @mysten-incubation/oc-memwal@${NEW_VERSION}" + cd packages/openclaw-memory-memwal + node -e "const p=require('./package.json');p.version='${NEW_VERSION}';require('fs').writeFileSync('./package.json',JSON.stringify(p,null,4)+'\n')" + npm publish --tag rc --provenance --access public + + # ── dev branch → prerelease (dev tag, auto-increment) ── + - name: Publish dev prerelease + if: github.ref == 'refs/heads/dev' + run: | + BASE_VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") + LATEST=$(npm view @mysten-incubation/oc-memwal versions --json 2>/dev/null \ + | node -p " + const versions = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); + const nums = (Array.isArray(versions) ? versions : [versions]) + .filter(v => v.startsWith('${BASE_VERSION}-dev.')) + .map(v => +v.split('-dev.')[1]) + .sort((a,b) => b - a); + nums.length ? nums[0] : -1; + " || echo "-1") + NEXT=$((LATEST + 1)) + NEW_VERSION="${BASE_VERSION}-dev.${NEXT}" + echo "Publishing @mysten-incubation/oc-memwal@${NEW_VERSION}" + cd packages/openclaw-memory-memwal + node -e "const p=require('./package.json');p.version='${NEW_VERSION}';require('fs').writeFileSync('./package.json',JSON.stringify(p,null,4)+'\n')" + npm publish --tag dev --provenance --access public diff --git a/docs/openclaw/how-it-works.md b/docs/openclaw/how-it-works.md index 6a99a5e9..02586441 100644 --- a/docs/openclaw/how-it-works.md +++ b/docs/openclaw/how-it-works.md @@ -20,7 +20,7 @@ graph TB LLM_PROC["Language Model\n(Gemini, GPT, Claude)"] end - subgraph "MemWal Server (TEE)" + subgraph "MemWal Relayer" SEARCH["Vector Search"] ANALYZE["Fact Extraction (LLM)"] STORE["Encrypted Storage"] @@ -56,7 +56,7 @@ graph TB | **Auto-recall hook** | Gateway (Node.js) | Searches MemWal before each turn, injects memories into prompt | | **Auto-capture hook** | Gateway (Node.js) | Extracts facts after each turn, stores via MemWal | | **Tool execution** | Gateway (Node.js) | Runs `memory_search` / `memory_store` when the LLM calls them | -| **MemWal Server** | Remote (TEE) | Handles vector search, LLM fact extraction, encrypted storage | +| **MemWal Relayer** | Remote | Handles vector search, LLM fact extraction, encrypted storage | | **Walrus** | Decentralized | Stores encrypted memory blobs | ## Message Flow diff --git a/docs/openclaw/quick-start.md b/docs/openclaw/quick-start.md index 86e485fc..2a6bd43c 100644 --- a/docs/openclaw/quick-start.md +++ b/docs/openclaw/quick-start.md @@ -1,6 +1,6 @@ --- title: "Quick Start" -description: "Install the MemWal memory plugin for OpenClaw and verify it works." +description: "Install the MemWal memory plugin for NemoClaw/OpenClaw and verify it works." --- Get the plugin running and test the memory loop in a few minutes. @@ -8,7 +8,6 @@ Get the plugin running and test the memory loop in a few minutes. ## Prerequisites - [OpenClaw](https://openclaw.ai) `>=2026.3.11` installed and running -- A package manager — [bun](https://bun.sh), [pnpm](https://pnpm.io), or [npm](https://www.npmjs.com) You'll also need a **delegate key**, **account ID**, and **relayer URL** from MemWal — the steps below will guide you through getting these. @@ -16,38 +15,10 @@ You'll also need a **delegate key**, **account ID**, and **relayer URL** from Me - ### Install dependencies - - - - ```bash - cd packages/openclaw-memory-memwal - bun install - ``` - - - ```bash - cd packages/openclaw-memory-memwal - pnpm install - ``` - - - ```bash - cd packages/openclaw-memory-memwal - npm install - ``` - - - - - - ### Link into OpenClaw - - OpenClaw discovers plugins from `~/.openclaw/extensions/`. Create a symlink: + ### Install the plugin ```bash - mkdir -p ~/.openclaw/extensions - ln -s "$(pwd)" ~/.openclaw/extensions/memory-memwal + openclaw plugins install @mysten-incubation/oc-memwal ``` @@ -95,9 +66,9 @@ You'll also need a **delegate key**, **account ID**, and **relayer URL** from Me ```jsonc { "plugins": { - "slots": { "memory": "memory-memwal" }, + "slots": { "memory": "oc-memwal" }, "entries": { - "memory-memwal": { + "oc-memwal": { "enabled": true, "config": { "privateKey": "${MEMWAL_PRIVATE_KEY}", // References the env var @@ -134,8 +105,8 @@ You'll also need a **delegate key**, **account ID**, and **relayer URL** from Me You should see in the logs: ``` - memory-memwal: registered (server: https://..., key: e21d...ed9b, namespace: default) - memory-memwal: connected (status: ok, version: ...) + oc-memwal: registered (server: https://..., key: e21d...ed9b, namespace: default) + oc-memwal: connected (status: ok, version: ...) ``` @@ -169,7 +140,7 @@ Bot: (responds normally) Check logs — you should see: ``` -memory-memwal: auto-captured 1 facts (agent: main, namespace: default) +oc-memwal: auto-captured 1 facts (agent: main, namespace: default) ``` **2. Recall it** — in a **new conversation**, ask about it: @@ -180,7 +151,7 @@ You: What programming languages do I like? Check logs — you should see: ``` -memory-memwal: auto-recall injected 1 memories (agent: main, namespace: default) +oc-memwal: auto-recall injected 1 memories (agent: main, namespace: default) ``` **3. Search from terminal** — confirm the memory exists via CLI: diff --git a/docs/openclaw/reference.md b/docs/openclaw/reference.md index 9f8605c4..2a5bb599 100644 --- a/docs/openclaw/reference.md +++ b/docs/openclaw/reference.md @@ -213,8 +213,8 @@ Full list of config options for `openclaw.json`: ### Plugin not loading -- Check the symlink exists: `ls -la ~/.openclaw/extensions/memory-memwal` -- Check that `openclaw.plugin.json` is in the package root +- Reinstall the plugin: `openclaw plugins install @mysten-incubation/oc-memwal` +- Check that `openclaw.plugin.json` exists in the installed extension - Restart the gateway after any config changes ### Health check failed diff --git a/packages/openclaw-memory-memwal/.gitignore b/packages/openclaw-memory-memwal/.gitignore new file mode 100644 index 00000000..849ddff3 --- /dev/null +++ b/packages/openclaw-memory-memwal/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/packages/openclaw-memory-memwal/README.md b/packages/openclaw-memory-memwal/README.md index c254c8ad..1b1c1024 100644 --- a/packages/openclaw-memory-memwal/README.md +++ b/packages/openclaw-memory-memwal/README.md @@ -1,4 +1,4 @@ -

@mysten-incubation/memory-memwal

+

@mysten-incubation/oc-memwal

Cloud-based long-term memory plugin for NemoClaw/OpenClaw — gives your AI agents persistent, encrypted, cross-session memory powered by MemWal. @@ -44,9 +44,9 @@ The plugin needs three values: | **Account ID** | Your MemWalAccount object ID on Sui (`0x...`) | | **Relayer URL** | The MemWal relayer endpoint that handles search, storage, and encryption | -Get your delegate key and account ID from the [MemWal dashboard](https://memwal.ai), or see the [Quick Start guide](/getting-started/quick-start) for detailed setup. +Get your delegate key and account ID from the [MemWal dashboard](https://memwal.ai), or see the [Quick Start guide](https://docs.memwal.ai/getting-started/quick-start) for detailed setup. -For the relayer, use a managed endpoint or [self-host your own](/relayer/self-hosting): +For the relayer, use a managed endpoint or [self-host your own](https://docs.memwal.ai/relayer/self-hosting): | Environment | Relayer URL | |-------------|-------------| @@ -55,15 +55,10 @@ For the relayer, use a managed endpoint or [self-host your own](/relayer/self-ho ## Quick Start -### 1. Install and link +### 1. Install ```bash -cd packages/openclaw-memory-memwal -bun install # or: pnpm install / npm install - -# Link into OpenClaw's extensions directory -mkdir -p ~/.openclaw/extensions -ln -s "$(pwd)" ~/.openclaw/extensions/memory-memwal +openclaw plugins install @mysten-incubation/oc-memwal ``` ### 2. Set your delegate key @@ -82,9 +77,9 @@ Add the plugin config to `~/.openclaw/openclaw.json`: ```jsonc { "plugins": { - "slots": { "memory": "memory-memwal" }, + "slots": { "memory": "oc-memwal" }, "entries": { - "memory-memwal": { + "oc-memwal": { "enabled": true, "config": { "privateKey": "${MEMWAL_PRIVATE_KEY}", // References the env var diff --git a/packages/openclaw-memory-memwal/openclaw.plugin.json b/packages/openclaw-memory-memwal/openclaw.plugin.json index e96fb644..d9a36710 100644 --- a/packages/openclaw-memory-memwal/openclaw.plugin.json +++ b/packages/openclaw-memory-memwal/openclaw.plugin.json @@ -57,12 +57,12 @@ "label": "Ed25519 Private Key", "sensitive": true, "placeholder": "64-character hex string or ${ENV_VAR}", - "help": "Get from app.memwal.com. Supports ${ENV_VAR} syntax." + "help": "Get from memwal.ai. Supports ${ENV_VAR} syntax." }, "accountId": { "label": "MemWalAccount ID", "placeholder": "0x3247e3da...", - "help": "MemWalAccount object ID on Sui. Get from app.memwal.com." + "help": "MemWalAccount object ID on Sui. Get from memwal.ai." }, "serverUrl": { "label": "MemWal Server URL", diff --git a/packages/openclaw-memory-memwal/package.json b/packages/openclaw-memory-memwal/package.json index 6be2e428..cfd13014 100644 --- a/packages/openclaw-memory-memwal/package.json +++ b/packages/openclaw-memory-memwal/package.json @@ -1,15 +1,40 @@ { - "name": "@mysten-incubation/memory-memwal", - "version": "0.2.0", - "private": true, + "name": "@mysten-incubation/oc-memwal", + "version": "0.0.1", "type": "module", - "description": "OpenClaw memory plugin — encrypted, decentralized long-term memory via MemWal + Walrus", + "description": "NemoClaw/OpenClaw memory plugin — encrypted, decentralized long-term memory via MemWal + Walrus", "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "openclaw.plugin.json", + "README.md" + ], + "scripts": { + "build": "tsc", + "typecheck": "tsc --noEmit", + "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", + "prepublishOnly": "npm run clean && npm run build" + }, + "engines": { + "node": ">=20.0.0" + }, "dependencies": { "@mysten-incubation/memwal": "^0.0.1", "@sinclair/typebox": "0.34.48", "zod": "^4.3.6" }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.7.0" + }, "peerDependencies": { "openclaw": ">=2026.3.11" }, @@ -20,7 +45,7 @@ }, "openclaw": { "extensions": [ - "./src/index.ts" + "./dist/index.js" ] } } diff --git a/packages/openclaw-memory-memwal/tsconfig.json b/packages/openclaw-memory-memwal/tsconfig.json new file mode 100644 index 00000000..c1142464 --- /dev/null +++ b/packages/openclaw-memory-memwal/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "verbatimModuleSyntax": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} From d70748de834bb6513f7ebb9d5dc433b6d064dc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=E1=BA=A7n=20Ph=E1=BA=A1m=20Minh=20H=C3=B9ng?= <46132442+hungtranphamminh@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:40:15 +0700 Subject: [PATCH 02/10] fix(openclaw-plugin): add repository field for npm OIDC provenance (#59) --- packages/openclaw-memory-memwal/package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/openclaw-memory-memwal/package.json b/packages/openclaw-memory-memwal/package.json index cfd13014..ef0237b5 100644 --- a/packages/openclaw-memory-memwal/package.json +++ b/packages/openclaw-memory-memwal/package.json @@ -4,6 +4,11 @@ "type": "module", "description": "NemoClaw/OpenClaw memory plugin — encrypted, decentralized long-term memory via MemWal + Walrus", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/MystenLabs/MemWal", + "directory": "packages/openclaw-memory-memwal" + }, "main": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { From 9663303c4ae1eb4c522678e1075db4bcdea18d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=E1=BA=A7n=20Ph=E1=BA=A1m=20Minh=20H=C3=B9ng?= <46132442+hungtranphamminh@users.noreply.github.com> Date: Mon, 30 Mar 2026 15:46:18 +0700 Subject: [PATCH 03/10] fix(openclaw-plugin): remove openclaw peer dep, downgrade zod to v3 (#60) Remove openclaw peer dependency that causes install failures via openclaw plugins install. Downgrade zod from v4 to v3 for monorepo compatibility. No API changes. --- packages/openclaw-memory-memwal/package.json | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/openclaw-memory-memwal/package.json b/packages/openclaw-memory-memwal/package.json index ef0237b5..176f1698 100644 --- a/packages/openclaw-memory-memwal/package.json +++ b/packages/openclaw-memory-memwal/package.json @@ -34,20 +34,12 @@ "dependencies": { "@mysten-incubation/memwal": "^0.0.1", "@sinclair/typebox": "0.34.48", - "zod": "^4.3.6" + "zod": "^3.23.0" }, "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.7.0" }, - "peerDependencies": { - "openclaw": ">=2026.3.11" - }, - "peerDependenciesMeta": { - "openclaw": { - "optional": true - } - }, "openclaw": { "extensions": [ "./dist/index.js" From b1ce9511c2c3821ed948a25122a81e78795250cc Mon Sep 17 00:00:00 2001 From: Ashwin-3cS Date: Sat, 28 Mar 2026 17:37:23 +0530 Subject: [PATCH 04/10] fix(server): use UTF-8 safe truncation in log preview strings Byte-level slicing via `&text[..text.len().min(50)]` panics when byte 50 lands inside a multi-byte UTF-8 character (e.g. emoji). Replace all four instances with a `truncate_str` helper that backs up to the nearest char boundary. --- services/server/src/routes.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/services/server/src/routes.rs b/services/server/src/routes.rs index ccaf5314..d33464ca 100644 --- a/services/server/src/routes.rs +++ b/services/server/src/routes.rs @@ -9,6 +9,19 @@ use crate::walrus; use crate::types::*; use crate::db::VectorDb; +/// Truncate a string to at most `max_bytes` bytes without splitting a UTF-8 +/// character. Falls back to the nearest char boundary when `max_bytes` lands +/// inside a multi-byte sequence (e.g. emoji). +fn truncate_str(s: &str, max_bytes: usize) -> &str { + if s.len() <= max_bytes { + return s; + } + let mut end = max_bytes; + while !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] +} // ============================================================ // Embedding — OpenRouter/OpenAI API (with mock fallback) @@ -119,7 +132,7 @@ pub async fn remember( let owner = &auth.owner; let text = &body.text; let namespace = &body.namespace; - tracing::info!("remember: text=\"{}...\" owner={} ns={}", &text[..text.len().min(50)], owner, namespace); + tracing::info!("remember: text=\"{}...\" owner={} ns={}", truncate_str(text, 50), owner, namespace); // Step 1: Embed text + SEAL encrypt concurrently (they're independent) let embed_fut = generate_embedding(&state.http_client, &state.config, text); @@ -178,7 +191,7 @@ pub async fn recall( // Owner is derived from delegate key via onchain verification (auth middleware) let owner = &auth.owner; let namespace = &body.namespace; - tracing::info!("recall: query=\"{}...\" owner={} ns={}", &body.query[..body.query.len().min(50)], owner, namespace); + tracing::info!("recall: query=\"{}...\" owner={} ns={}", truncate_str(&body.query, 50), owner, namespace); // Use delegate key from SDK for SEAL decryption (falls back to server key) let private_key = auth.delegate_key.as_deref() @@ -368,7 +381,7 @@ pub async fn analyze( let owner = &auth.owner; let namespace = &body.namespace; - tracing::info!("analyze: text=\"{}...\" owner={} ns={}", &body.text[..body.text.len().min(50)], owner, namespace); + tracing::info!("analyze: text=\"{}...\" owner={} ns={}", truncate_str(&body.text, 50), owner, namespace); // Step 1: Extract facts using LLM let facts = extract_facts_llm(&state.http_client, &state.config, &body.text).await?; @@ -590,7 +603,7 @@ pub async fn ask( let owner = &auth.owner; let namespace = &body.namespace; let limit = body.limit.unwrap_or(5); - tracing::info!("ask: question=\"{}...\" owner={} ns={}", &body.question[..body.question.len().min(50)], owner, namespace); + tracing::info!("ask: question=\"{}...\" owner={} ns={}", truncate_str(&body.question, 50), owner, namespace); // Step 1: Recall relevant memories let query_vector = generate_embedding(&state.http_client, &state.config, &body.question).await?; From be4e439d3ad4d5fdfee7cd91c22b511318efd075 Mon Sep 17 00:00:00 2001 From: ducnmm Date: Tue, 31 Mar 2026 08:18:18 +0700 Subject: [PATCH 05/10] feat(relayer): multi-layer rate limiting with Redis, storage quota, and cost-weighted endpoints [ENG-1081] --- services/server/Cargo.lock | 87 ++++- services/server/Cargo.toml | 3 + .../server/migrations/003_rate_limiter.sql | 13 + services/server/scripts/sidecar-server.ts | 2 +- services/server/src/db.rs | 33 +- services/server/src/main.rs | 18 + services/server/src/rate_limit.rs | 326 ++++++++++++++++++ services/server/src/routes.rs | 34 +- services/server/src/types.rs | 15 + 9 files changed, 519 insertions(+), 12 deletions(-) create mode 100644 services/server/migrations/003_rate_limiter.sql create mode 100644 services/server/src/rate_limit.rs diff --git a/services/server/Cargo.lock b/services/server/Cargo.lock index 8ef8da12..980c8c2b 100644 --- a/services/server/Cargo.lock +++ b/services/server/Cargo.lock @@ -32,6 +32,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -205,6 +214,20 @@ dependencies = [ "windows-link", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -842,7 +865,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "system-configuration", "tokio", "tower-service", @@ -1010,6 +1033,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1145,6 +1177,7 @@ dependencies = [ "futures", "hex", "pgvector", + "redis", "reqwest", "serde", "serde_json", @@ -1212,6 +1245,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -1488,6 +1531,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "redis" +version = "0.27.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "combine", + "futures-util", + "itertools", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.5.10", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1787,6 +1854,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1848,6 +1921,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -2247,7 +2330,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] diff --git a/services/server/Cargo.toml b/services/server/Cargo.toml index 44c69541..004b6fd4 100644 --- a/services/server/Cargo.toml +++ b/services/server/Cargo.toml @@ -42,6 +42,9 @@ walrus_rs = "0.1" # Async utilities futures = "0.3" +# Rate limiting (Redis-backed) +redis = { version = "0.27", features = ["tokio-comp"] } + # Utils uuid = { version = "1", features = ["v4"] } chrono = "0.4" diff --git a/services/server/migrations/003_rate_limiter.sql b/services/server/migrations/003_rate_limiter.sql new file mode 100644 index 00000000..12cb101f --- /dev/null +++ b/services/server/migrations/003_rate_limiter.sql @@ -0,0 +1,13 @@ +-- memwal — Storage Quota Tracking +-- Rate limiting is handled by Redis (no PostgreSQL table needed). +-- Storage quota is tracked per-row in vector_entries. + +-- ============================================================ +-- Storage quota: track blob size in vector_entries +-- ============================================================ +-- blob_size_bytes tracks the size of each encrypted blob uploaded. +-- Total storage per user = SUM(blob_size_bytes) WHERE owner = $1. +-- When blobs expire and are cleaned up (delete_by_blob_id), quota +-- is automatically reduced. +ALTER TABLE vector_entries + ADD COLUMN IF NOT EXISTS blob_size_bytes BIGINT NOT NULL DEFAULT 0; diff --git a/services/server/scripts/sidecar-server.ts b/services/server/scripts/sidecar-server.ts index 19220ef1..1d35a487 100644 --- a/services/server/scripts/sidecar-server.ts +++ b/services/server/scripts/sidecar-server.ts @@ -51,7 +51,7 @@ const WALRUS_UPLOAD_RELAY_URL = process.env.WALRUS_UPLOAD_RELAY_URL || ( : "https://upload-relay.mainnet.walrus.space" ); -const DEFAULT_WALRUS_EPOCHS = SUI_NETWORK === "testnet" ? 50 : 2; +const DEFAULT_WALRUS_EPOCHS = SUI_NETWORK === "testnet" ? 50 : 3; const suiClient = new SuiJsonRpcClient({ url: getJsonRpcFullnodeUrl(SUI_NETWORK), diff --git a/services/server/src/db.rs b/services/server/src/db.rs index 97d2e7fc..02688217 100644 --- a/services/server/src/db.rs +++ b/services/server/src/db.rs @@ -30,12 +30,18 @@ impl VectorDb { .await .map_err(|e| AppError::Internal(format!("Failed to run migration 002: {}", e)))?; + let migration_003 = include_str!("../migrations/003_rate_limiter.sql"); + sqlx::raw_sql(migration_003) + .execute(&pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to run migration 003: {}", e)))?; + tracing::info!("database connected and migrations applied"); Ok(Self { pool }) } - /// Insert a vector entry + /// Insert a vector entry (with blob size tracking for storage quota) pub async fn insert_vector( &self, id: &str, @@ -43,23 +49,25 @@ impl VectorDb { namespace: &str, blob_id: &str, vector: &[f32], + blob_size_bytes: i64, ) -> Result<(), AppError> { let embedding = Vector::from(vector.to_vec()); sqlx::query( - "INSERT INTO vector_entries (id, owner, namespace, blob_id, embedding) - VALUES ($1, $2, $3, $4, $5)", + "INSERT INTO vector_entries (id, owner, namespace, blob_id, embedding, blob_size_bytes) + VALUES ($1, $2, $3, $4, $5, $6)", ) .bind(id) .bind(owner) .bind(namespace) .bind(blob_id) .bind(embedding) + .bind(blob_size_bytes) .execute(&self.pool) .await .map_err(|e| AppError::Internal(format!("Failed to insert vector: {}", e)))?; - tracing::debug!("inserted vector: id={}, blob_id={}, owner={}, ns={}", id, blob_id, owner, namespace); + tracing::debug!("inserted vector: id={}, blob_id={}, owner={}, ns={}, size={}B", id, blob_id, owner, namespace, blob_size_bytes); Ok(()) } @@ -198,6 +206,23 @@ impl VectorDb { Ok(()) } + // ============================================================ + // Storage Quota (still PostgreSQL — tracks per-row blob sizes) + // ============================================================ + + /// Get total storage used by a user (sum of blob_size_bytes for active entries). + pub async fn get_storage_used(&self, owner: &str) -> Result { + let row: (i64,) = sqlx::query_as( + "SELECT COALESCE(SUM(blob_size_bytes)::BIGINT, 0) FROM vector_entries WHERE owner = $1", + ) + .bind(owner) + .fetch_one(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to get storage used: {}", e)))?; + + Ok(row.0) + } + // ============================================================ // Accounts (populated by v2-indexer) // ============================================================ diff --git a/services/server/src/main.rs b/services/server/src/main.rs index e90669d5..d0b883fd 100644 --- a/services/server/src/main.rs +++ b/services/server/src/main.rs @@ -1,5 +1,6 @@ mod auth; mod db; +mod rate_limit; mod routes; mod seal; mod sui; @@ -34,6 +35,12 @@ async fn main() { tracing::info!(" package id: {}", config.package_id); tracing::info!(" registry id: {}", config.registry_id); tracing::info!(" memwal account: {}", config.memwal_account_id.as_deref().unwrap_or("(from client header)")); + tracing::info!(" rate limit: burst={}/min, sustained={}/hr, per-key={}/min, quota={}MB/user", + config.rate_limit.max_requests_per_minute, + config.rate_limit.max_requests_per_hour, + config.rate_limit.max_requests_per_delegate_key, + config.rate_limit.max_storage_bytes / 1_048_576 + ); // Start TS sidecar HTTP server (SEAL + Walrus operations) let sidecar_url = config.sidecar_url.clone(); @@ -98,6 +105,12 @@ async fn main() { // Build key pool for parallel Walrus uploads let key_pool = KeyPool::new(config.sui_private_keys.clone()); + // Initialize Redis for rate limiting + let redis = rate_limit::create_redis_client(&config.rate_limit.redis_url) + .await + .expect("Failed to connect to Redis for rate limiting"); + tracing::info!(" Redis: connected at {}", config.rate_limit.redis_url); + // Shared application state let state = Arc::new(AppState { db, @@ -105,6 +118,7 @@ async fn main() { http_client, walrus_client, key_pool, + redis, }); // Build routes @@ -118,6 +132,10 @@ async fn main() { .route("/api/analyze", post(routes::analyze)) .route("/api/ask", post(routes::ask)) .route("/api/restore", post(routes::restore)) + .layer(middleware::from_fn_with_state( + state.clone(), + rate_limit::rate_limit_middleware, + )) .layer(middleware::from_fn_with_state( state.clone(), auth::verify_signature, diff --git a/services/server/src/rate_limit.rs b/services/server/src/rate_limit.rs new file mode 100644 index 00000000..e9dc490b --- /dev/null +++ b/services/server/src/rate_limit.rs @@ -0,0 +1,326 @@ +use axum::{ + extract::{Request, State}, + http::StatusCode, + middleware::Next, + response::Response, +}; +use std::sync::Arc; + +use crate::types::{AppError, AppState}; + +// ============================================================ +// Rate Limit Configuration +// ============================================================ + +#[derive(Debug, Clone)] +pub struct RateLimitConfig { + // --- Per-account burst window --- + /// Maximum weighted requests per minute per user (default: 60) + pub max_requests_per_minute: i64, + + // --- Per-account sustained window --- + /// Maximum weighted requests per hour per user (default: 500) + pub max_requests_per_hour: i64, + + // --- Per-delegate-key window --- + /// Maximum weighted requests per minute per delegate key (default: 30) + pub max_requests_per_delegate_key: i64, + + // --- Storage quota --- + /// Maximum storage per user in bytes (default: 1 GB) + pub max_storage_bytes: i64, + + /// Redis URL (default: redis://localhost:6379) + pub redis_url: String, +} + +impl Default for RateLimitConfig { + fn default() -> Self { + Self { + max_requests_per_minute: 60, + max_requests_per_hour: 500, + max_requests_per_delegate_key: 30, + max_storage_bytes: 1_073_741_824, // 1 GB + redis_url: "redis://localhost:6379".to_string(), + } + } +} + +impl RateLimitConfig { + pub fn from_env() -> Self { + let mut config = Self::default(); + + if let Ok(val) = std::env::var("RATE_LIMIT_REQUESTS_PER_MINUTE") { + if let Ok(n) = val.parse::() { + config.max_requests_per_minute = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_REQUESTS_PER_HOUR") { + if let Ok(n) = val.parse::() { + config.max_requests_per_hour = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_DELEGATE_KEY_PER_MINUTE") { + if let Ok(n) = val.parse::() { + config.max_requests_per_delegate_key = n; + } + } + + if let Ok(val) = std::env::var("RATE_LIMIT_STORAGE_BYTES") { + if let Ok(n) = val.parse::() { + config.max_storage_bytes = n; + } + } + + if let Ok(val) = std::env::var("REDIS_URL") { + config.redis_url = val; + } + + config + } +} + +// ============================================================ +// Cost Weights — per endpoint +// ============================================================ + +/// Get the cost weight for a given API path. +/// +/// Expensive endpoints (embedding + encrypt + Walrus upload + LLM) +/// consume more of the rate limit budget than cheap read endpoints. +fn endpoint_weight(path: &str) -> i64 { + match path { + "/api/analyze" => 10, // LLM extract + N × (embed + encrypt + upload) + "/api/remember" => 5, // embed + SEAL encrypt + Walrus upload + "/api/remember/manual" => 3, // Walrus upload only (client did embed/encrypt) + "/api/restore" => 3, // download + decrypt + re-embed + "/api/ask" => 2, // recall + LLM + _ => 1, // recall, recall/manual, etc. + } +} + +// ============================================================ +// Redis Client +// ============================================================ + +/// Create a Redis multiplexed connection for shared use across the app. +pub async fn create_redis_client(redis_url: &str) -> Result { + let client = redis::Client::open(redis_url) + .map_err(|e| format!("Failed to create Redis client: {}", e))?; + + let conn = client.get_multiplexed_async_connection() + .await + .map_err(|e| format!("Failed to connect to Redis: {}", e))?; + + Ok(conn) +} + +// ============================================================ +// Sliding Window Helpers +// ============================================================ + +/// Check the current count in a Redis sorted set sliding window. +/// Returns the count of entries within the window. +async fn check_window( + redis: &mut redis::aio::MultiplexedConnection, + key: &str, + window_start: f64, +) -> Result { + let result: ((), i64) = redis::pipe() + .atomic() + .zrembyscore(key, 0.0_f64, window_start) + .zcard(key) + .query_async(redis) + .await?; + + Ok(result.1) +} + +/// Record weighted entries in a Redis sorted set sliding window. +/// Adds `weight` entries and sets TTL. +async fn record_in_window( + redis: &mut redis::aio::MultiplexedConnection, + key: &str, + now: f64, + weight: i64, + ttl_seconds: i64, +) { + let mut pipe = redis::pipe(); + for i in 0..weight { + // Use fractional offsets to create unique members + let ts = now + i as f64 * 0.001; + pipe.zadd(key, ts, format!("{}", ts)); + } + pipe.expire(key, ttl_seconds); + + let _: Result<(), _> = pipe.query_async(redis).await; +} + +// ============================================================ +// Rate Limit Response +// ============================================================ + +/// Build a 429 response with JSON body and Retry-After header. +fn rate_limit_response(layer: &str, limit: i64, window: &str, retry_after: u64) -> Response { + let body = serde_json::json!({ + "error": "Rate limit exceeded", + "layer": layer, + "limit": format!("{} weighted-requests/{}", limit, window), + "retry_after_seconds": retry_after, + }); + + axum::response::Response::builder() + .status(StatusCode::TOO_MANY_REQUESTS) + .header("Content-Type", "application/json") + .header("Retry-After", retry_after.to_string()) + .body(axum::body::Body::from(serde_json::to_string(&body).unwrap())) + .unwrap() +} + +// ============================================================ +// Rate Limit Middleware +// ============================================================ + +/// Multi-layer rate limiting middleware for authenticated routes. +/// +/// Checks 3 layers (all must pass): +/// 1. Per-delegate-key: 30 weighted-req/min (prevents compromised key abuse) +/// 2. Per-account burst: 60 weighted-req/min (prevents spam) +/// 3. Per-account sustained: 500 weighted-req/hour (prevents slow-burn) +/// +/// Endpoints are cost-weighted: +/// analyze=10, remember=5, remember/manual=3, restore=3, ask=2, recall=1 +/// +/// Returns 429 Too Many Requests with JSON body if any layer exceeds its limit. +pub async fn rate_limit_middleware( + State(state): State>, + request: Request, + next: Next, +) -> Response { + // Extract auth info (set by auth middleware) + let auth_info = request + .extensions() + .get::() + .cloned(); + + let auth = match auth_info { + Some(a) => a, + None => { + // No auth info = not an authenticated route, skip rate limiting + return next.run(request).await; + } + }; + + let config = &state.config.rate_limit; + let mut redis = state.redis.clone(); + let now = chrono::Utc::now().timestamp_millis() as f64; + + // Determine cost weight based on endpoint + let weight = endpoint_weight(request.uri().path()); + + // --- Layer 1: Per-delegate-key (burst) --- + let dk_key = format!("rate:dk:{}", auth.public_key); + let dk_window_start = now - 60_000.0; // 1 min window + + match check_window(&mut redis, &dk_key, dk_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_delegate_key { + tracing::warn!( + "rate limit [delegate-key]: key={}... count={}/{} weight={} path={}", + &auth.public_key[..16], count, + config.max_requests_per_delegate_key, weight, request.uri().path() + ); + return rate_limit_response("delegate_key", config.max_requests_per_delegate_key, "min", 60); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (dk): {}, allowing", e); + } + } + + // --- Layer 2: Per-account burst (1 min) --- + let burst_key = format!("rate:{}", auth.owner); + let burst_window_start = now - 60_000.0; + + match check_window(&mut redis, &burst_key, burst_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_minute { + tracing::warn!( + "rate limit [burst]: owner={} count={}/{} weight={} path={}", + auth.owner, count, config.max_requests_per_minute, weight, request.uri().path() + ); + return rate_limit_response("account_burst", config.max_requests_per_minute, "min", 60); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (burst): {}, allowing", e); + } + } + + // --- Layer 3: Per-account sustained (1 hour) --- + let hourly_key = format!("rate:hr:{}", auth.owner); + let hourly_window_start = now - 3_600_000.0; + + match check_window(&mut redis, &hourly_key, hourly_window_start).await { + Ok(count) => { + if count >= config.max_requests_per_hour { + tracing::warn!( + "rate limit [sustained]: owner={} count={}/{} weight={} path={}", + auth.owner, count, config.max_requests_per_hour, weight, request.uri().path() + ); + return rate_limit_response("account_sustained", config.max_requests_per_hour, "hour", 300); + } + } + Err(e) => { + tracing::error!("redis rate limit check failed (sustained): {}, allowing", e); + } + } + + // --- All checks passed: record weighted entries in all 3 windows --- + record_in_window(&mut redis, &dk_key, now, weight, 120).await; // TTL 2min + record_in_window(&mut redis, &burst_key, now + 0.1, weight, 120).await; // offset to avoid collision + record_in_window(&mut redis, &hourly_key, now + 0.2, weight, 3700).await; // TTL ~1hr+buffer + + next.run(request).await +} + +// ============================================================ +// Storage Quota Check (called from routes, not middleware) +// ============================================================ + +/// Check if a user has enough storage quota for a new blob. +/// +/// Storage tracking still uses PostgreSQL (it's per-row in vector_entries). +/// Returns `Ok(())` if within quota, `Err(AppError::QuotaExceeded)` if not. +pub async fn check_storage_quota( + state: &AppState, + owner: &str, + additional_bytes: i64, +) -> Result<(), AppError> { + let max_bytes = state.config.rate_limit.max_storage_bytes; + + // 0 or negative means unlimited + if max_bytes <= 0 { + return Ok(()); + } + + let used = state.db.get_storage_used(owner).await?; + let projected = used + additional_bytes; + + if projected > max_bytes { + let used_mb = used as f64 / 1_048_576.0; + let max_mb = max_bytes as f64 / 1_048_576.0; + tracing::warn!( + "storage quota exceeded: owner={} used={:.1}MB + {:.1}MB > max={:.1}MB", + owner, used_mb, additional_bytes as f64 / 1_048_576.0, max_mb + ); + return Err(AppError::QuotaExceeded(format!( + "Storage quota exceeded: {:.1}MB used of {:.1}MB allowed", + used_mb, max_mb + ))); + } + + Ok(()) +} diff --git a/services/server/src/routes.rs b/services/server/src/routes.rs index ccaf5314..7c32facb 100644 --- a/services/server/src/routes.rs +++ b/services/server/src/routes.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use crate::seal; use crate::walrus; +use crate::rate_limit; use crate::types::*; use crate::db::VectorDb; @@ -121,6 +122,10 @@ pub async fn remember( let namespace = &body.namespace; tracing::info!("remember: text=\"{}...\" owner={} ns={}", &text[..text.len().min(50)], owner, namespace); + // Check storage quota before processing + let text_bytes = text.as_bytes().len() as i64; + rate_limit::check_storage_quota(&state, owner, text_bytes).await?; + // Step 1: Embed text + SEAL encrypt concurrently (they're independent) let embed_fut = generate_embedding(&state.http_client, &state.config, text); let encrypt_fut = seal::seal_encrypt( @@ -142,8 +147,9 @@ pub async fn remember( let blob_id = upload_result.blob_id; // Step 3: Store {vector, blobId, namespace} in Vector DB + let blob_size = encrypted.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, &blob_id, &vector).await?; + state.db.insert_vector(&id, owner, namespace, &blob_id, &vector, blob_size).await?; tracing::info!( "remember complete: blob_id={}, owner={}, ns={}, dims={}", @@ -231,7 +237,15 @@ pub async fn recall( } } Err(e) => { - tracing::warn!("Failed to SEAL decrypt blob {}: {}", blob_id, e); + let err_str = e.to_string(); + let is_permanent = err_str.contains("Not enough shares") + || err_str.contains("decrypt failed"); + if is_permanent { + tracing::warn!("SEAL decrypt permanently failed for blob {}, cleaning up: {}", blob_id, e); + cleanup_expired_blob(db, &blob_id).await; + } else { + tracing::warn!("Failed to SEAL decrypt blob {}: {}", blob_id, e); + } None } } @@ -283,6 +297,9 @@ pub async fn remember_manual( .decode(&body.encrypted_data) .map_err(|e| AppError::BadRequest(format!("encrypted_data is not valid base64: {}", e)))?; + // Check storage quota before upload + rate_limit::check_storage_quota(&state, owner, encrypted_bytes.len() as i64).await?; + // Upload encrypted bytes to Walrus via sidecar (server pays gas) let sui_key = state.config.sui_private_key.as_deref().ok_or_else(|| { AppError::Internal("SERVER_SUI_PRIVATE_KEY not configured for Walrus upload".into()) @@ -304,8 +321,9 @@ pub async fn remember_manual( tracing::info!("remember_manual: walrus upload ok blob_id={}", blob_id); // Store {vector, blobId, namespace} in Vector DB + let blob_size = encrypted_bytes.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, &blob_id, &body.vector).await?; + state.db.insert_vector(&id, owner, namespace, &blob_id, &body.vector, blob_size).await?; tracing::info!("remember_manual complete: id={}, blob_id={}, ns={}", id, blob_id, namespace); @@ -382,6 +400,10 @@ pub async fn analyze( })); } + // Check storage quota before processing all facts + let total_text_bytes: i64 = facts.iter().map(|f| f.as_bytes().len() as i64).sum(); + rate_limit::check_storage_quota(&state, owner, total_text_bytes).await?; + // Step 2: Process all facts concurrently (embed + encrypt → upload → store) // Each fact gets its own key from the pool so sidecar can upload them in parallel // (different signer addresses bypass the per-signer serialization lock). @@ -414,8 +436,9 @@ pub async fn analyze( ).await?; // Store in Vector DB with namespace + let blob_size = encrypted.len() as i64; let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, &owner, &namespace, &upload_result.blob_id, &vector).await?; + state.db.insert_vector(&id, &owner, &namespace, &upload_result.blob_id, &vector, blob_size).await?; Ok::(AnalyzedFact { text: fact_text, @@ -935,7 +958,8 @@ pub async fn restore( let restored = results.len(); for (blob_id, vector) in &results { let id = uuid::Uuid::new_v4().to_string(); - state.db.insert_vector(&id, owner, namespace, blob_id, vector).await?; + // Restore flow: blob_size not tracked (already counted when first stored) + state.db.insert_vector(&id, owner, namespace, blob_id, vector, 0).await?; } tracing::info!( diff --git a/services/server/src/types.rs b/services/server/src/types.rs index 8125e95d..ad82cb8f 100644 --- a/services/server/src/types.rs +++ b/services/server/src/types.rs @@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use serde::{Deserialize, Serialize}; use crate::db::VectorDb; +use crate::rate_limit::RateLimitConfig; // ============================================================ // App State (shared across routes + middleware) @@ -15,6 +16,8 @@ pub struct AppState { pub walrus_client: walrus_rs::WalrusClient, /// Round-robin pool of Sui private keys for parallel Walrus uploads pub key_pool: KeyPool, + /// Redis multiplexed connection for rate limiting + pub redis: redis::aio::MultiplexedConnection, } // ============================================================ @@ -75,6 +78,8 @@ pub struct Config { pub registry_id: String, /// URL of the SEAL/Walrus TS sidecar HTTP server pub sidecar_url: String, + /// Rate limiting configuration + pub rate_limit: RateLimitConfig, } impl Config { @@ -124,6 +129,7 @@ impl Config { .expect("MEMWAL_REGISTRY_ID must be set"), sidecar_url: std::env::var("SIDECAR_URL") .unwrap_or_else(|_| "http://localhost:9000".to_string()), + rate_limit: RateLimitConfig::from_env(), } } } @@ -332,6 +338,11 @@ pub enum AppError { Internal(String), /// Walrus blob not found (expired or deleted) — triggers cleanup BlobNotFound(String), + /// Rate limit exceeded (HTTP 429) + #[allow(dead_code)] + RateLimited(String), + /// Storage quota exceeded (HTTP 402) + QuotaExceeded(String), } impl std::fmt::Display for AppError { @@ -341,6 +352,8 @@ impl std::fmt::Display for AppError { AppError::Unauthorized(msg) => write!(f, "Unauthorized: {}", msg), AppError::Internal(msg) => write!(f, "Internal Error: {}", msg), AppError::BlobNotFound(msg) => write!(f, "Blob Not Found: {}", msg), + AppError::RateLimited(msg) => write!(f, "Rate Limited: {}", msg), + AppError::QuotaExceeded(msg) => write!(f, "Quota Exceeded: {}", msg), } } } @@ -355,6 +368,8 @@ impl axum::response::IntoResponse for AppError { msg.clone(), ), AppError::BlobNotFound(msg) => (axum::http::StatusCode::NOT_FOUND, msg.clone()), + AppError::RateLimited(msg) => (axum::http::StatusCode::TOO_MANY_REQUESTS, msg.clone()), + AppError::QuotaExceeded(msg) => (axum::http::StatusCode::PAYMENT_REQUIRED, msg.clone()), }; let body = serde_json::json!({ "error": message }); From 71fcd0b7935ca3d7da0ae80e5bbd4b2eac0676f7 Mon Sep 17 00:00:00 2001 From: ducnmm Date: Tue, 31 Mar 2026 13:15:34 +0700 Subject: [PATCH 06/10] docs: improve docs for AI agents, add SKILL.md, llms.txt, changelogs, setup guide, and release automation --- .github/workflows/release-oc-memwal.yml | 55 ++- .github/workflows/release-sdk.yml | 58 ++- README.md | 28 +- SKILL.md | 253 +++++++++++ docs/contributing/run-repo-locally.md | 129 +++++- docs/docs.json | 6 +- docs/llms-full.txt | 452 +++++++++++++++++++ docs/llms.txt | 58 +++ docs/openclaw/changelog.md | 19 + docs/sdk/changelog.md | 19 + packages/openclaw-memory-memwal/CHANGELOG.md | 12 + packages/sdk/CHANGELOG.md | 12 + 12 files changed, 1070 insertions(+), 31 deletions(-) create mode 100644 SKILL.md create mode 100644 docs/llms-full.txt create mode 100644 docs/llms.txt create mode 100644 docs/openclaw/changelog.md create mode 100644 docs/sdk/changelog.md create mode 100644 packages/openclaw-memory-memwal/CHANGELOG.md create mode 100644 packages/sdk/CHANGELOG.md diff --git a/.github/workflows/release-oc-memwal.yml b/.github/workflows/release-oc-memwal.yml index 9994accc..3c998675 100644 --- a/.github/workflows/release-oc-memwal.yml +++ b/.github/workflows/release-oc-memwal.yml @@ -51,14 +51,61 @@ jobs: - name: Build plugin run: cd packages/openclaw-memory-memwal && npm run build - # ── main branch → stable release (latest) ── + # ── main branch → changeset version + stable release (latest) ── + - name: Apply changesets (update version & CHANGELOG) + if: github.ref == 'refs/heads/main' + id: changeset_version + run: | + pnpm changeset version 2>/dev/null || true + if git diff --quiet; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Commit changelog & version bump + if: github.ref == 'refs/heads/main' && steps.changeset_version.outputs.has_changes == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "chore: version packages & update changelog [skip ci]" + git push + - name: Publish stable release if: github.ref == 'refs/heads/main' + id: publish_oc run: | - BASE_VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") - npm view @mysten-incubation/oc-memwal@$BASE_VERSION version 2>/dev/null \ - && echo "Version $BASE_VERSION already published, skipping" && exit 0 + VERSION=$(node -p "require('./packages/openclaw-memory-memwal/package.json').version") + echo "version=$VERSION" >> $GITHUB_OUTPUT + npm view @mysten-incubation/oc-memwal@$VERSION version 2>/dev/null \ + && echo "Version $VERSION already published, skipping" && echo "published=false" >> $GITHUB_OUTPUT && exit 0 cd packages/openclaw-memory-memwal && npm publish --provenance --access public + echo "published=true" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + if: github.ref == 'refs/heads/main' && steps.publish_oc.outputs.published == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const version = '${{ steps.publish_oc.outputs.version }}'; + const tag = `@mysten-incubation/oc-memwal@${version}`; + let body = `Release @mysten-incubation/oc-memwal v${version}`; + try { + const changelog = fs.readFileSync('packages/openclaw-memory-memwal/CHANGELOG.md', 'utf8'); + const match = changelog.match(/## \d+\.\d+\.\d+[\s\S]*?(?=## \d+\.\d+\.\d+|$)/); + if (match) body = match[0].trim(); + } catch (e) { /* use default body */ } + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: tag, + body, + draft: false, + prerelease: false, + }); # ── staging branch → release candidate (rc tag, auto-increment) ── - name: Publish staging release candidate diff --git a/.github/workflows/release-sdk.yml b/.github/workflows/release-sdk.yml index 401414f5..31bf3a9d 100644 --- a/.github/workflows/release-sdk.yml +++ b/.github/workflows/release-sdk.yml @@ -48,14 +48,64 @@ jobs: - name: Build SDK run: pnpm build:sdk - # ── main branch → stable release (latest) ── + # ── main branch → changeset version + stable release (latest) ── + - name: Apply changesets (update version & CHANGELOG) + if: github.ref == 'refs/heads/main' + id: changeset_version + run: | + # Consume pending changesets → bump version + update CHANGELOG.md + pnpm changeset version 2>/dev/null || true + # Check if changeset produced any changes + if git diff --quiet; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Commit changelog & version bump + if: github.ref == 'refs/heads/main' && steps.changeset_version.outputs.has_changes == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "chore: version packages & update changelog [skip ci]" + git push + - name: Publish stable release if: github.ref == 'refs/heads/main' + id: publish_sdk run: | - BASE_VERSION=$(node -p "require('./packages/sdk/package.json').version") - npm view @mysten-incubation/memwal@$BASE_VERSION version 2>/dev/null \ - && echo "Version $BASE_VERSION already published, skipping" && exit 0 + VERSION=$(node -p "require('./packages/sdk/package.json').version") + echo "version=$VERSION" >> $GITHUB_OUTPUT + npm view @mysten-incubation/memwal@$VERSION version 2>/dev/null \ + && echo "Version $VERSION already published, skipping" && echo "published=false" >> $GITHUB_OUTPUT && exit 0 cd packages/sdk && npm publish --provenance --access public + echo "published=true" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + if: github.ref == 'refs/heads/main' && steps.publish_sdk.outputs.published == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const version = '${{ steps.publish_sdk.outputs.version }}'; + const tag = `@mysten-incubation/memwal@${version}`; + // Extract latest changelog entry + let body = `Release @mysten-incubation/memwal v${version}`; + try { + const changelog = fs.readFileSync('packages/sdk/CHANGELOG.md', 'utf8'); + const match = changelog.match(/## \d+\.\d+\.\d+[\s\S]*?(?=## \d+\.\d+\.\d+|$)/); + if (match) body = match[0].trim(); + } catch (e) { /* use default body */ } + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: tag, + body, + draft: false, + prerelease: false, + }); # ── staging branch → release candidate (rc tag, auto-increment) ── - name: Publish staging release candidate diff --git a/README.md b/README.md index d482738f..28122bc3 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ retrieving them with semantic search. > MemWal is currently in beta and actively evolving. While fully usable today, we continue to refine the developer experience and operational guidance. We welcome feedback from early builders as we continue to improve the product. +## For AI Agents + +- **Single-file guide**: Read [`SKILL.md`](SKILL.md) for a complete integration reference (install, configure, API surface, troubleshooting) +- **LLM-friendly docs**: [`llms.txt`](https://docs.memwal.ai/llms.txt) — structured overview following the [llmstxt.org](https://llmstxt.org) standard +- **Full context**: [`llms-full.txt`](https://docs.memwal.ai/llms-full.txt) — expanded version with inlined page content + ## Install ```bash @@ -60,7 +66,13 @@ From the repository root: pnpm install ``` -Then start the surface you need, for example: +> **Important**: Build the SDK first — apps depend on its compiled output. + +```bash +pnpm build:sdk +``` + +Then start the surface you need: ```bash pnpm dev:app @@ -69,7 +81,7 @@ pnpm dev:chatbot pnpm dev:researcher ``` -For broader local setup guidance, see: +For the full step-by-step setup guide, see: - [Run the Repo Locally](docs/contributing/run-repo-locally.md) @@ -81,6 +93,18 @@ For broader local setup guidance, see: | `@mysten-incubation/memwal/manual` | Manual client flow (`MemWalManual`). You handle embedding calls and local SEAL operations. The relayer still handles upload relay, registration, search, and restore. | | `@mysten-incubation/memwal/ai` | Vercel AI SDK integration - wraps `MemWal` as middleware for use with `streamText`, `generateText`, etc. | +## OpenClaw / NemoClaw Plugin + +[`@mysten-incubation/oc-memwal`](packages/openclaw-memory-memwal) — a memory plugin for [OpenClaw](https://openclaw.ai) agents. It gives OpenClaw persistent, encrypted memory via MemWal with automatic recall and capture hooks. + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +- [Plugin Quick Start](docs/openclaw/quick-start.md) +- [How It Works](docs/openclaw/how-it-works.md) +- [Reference](docs/openclaw/reference.md) + ## How It Works 1. **Scope** - Each memory operation runs inside an `owner + namespace` boundary diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 00000000..00cad692 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,253 @@ +--- +name: memwal +version: 0.0.1 +description: | + Privacy-first AI memory SDK for decentralized storage on Sui blockchain with Walrus. + + Use when users say: + - "add memory to my app" + - "store encrypted memories" + - "integrate MemWal" + - "AI agent memory" + - "persistent memory SDK" + - "Walrus memory storage" + - "setup MemWal" + - "recall memories" + +keywords: + - memwal + - memory sdk + - ai memory + - encrypted memory + - walrus storage + - sui blockchain + - delegate key + - semantic search + - vercel ai sdk +--- + +# MemWal — Privacy-First AI Memory SDK + +MemWal is a TypeScript SDK for persistent, encrypted AI memory. It stores memories on Walrus (decentralized storage), encrypts them with SEAL, enforces ownership onchain via Sui smart contracts, and retrieves them with semantic (vector) search. Memories are scoped by `owner + namespace` — each namespace is an isolated memory space. + +--- + +## When to Use + +Use MemWal when your app or agent needs: + +- **Persistent memory** across sessions, devices, or restarts +- **Encrypted storage** — end-to-end encryption, only the owner and authorized delegates can decrypt +- **Semantic recall** — retrieve memories by meaning, not just keywords +- **Decentralized storage** — no single point of failure, stored on Walrus +- **Onchain ownership** — cryptographically enforced access control on Sui +- **Cross-app memory** — share memory between apps via delegate keys + +--- + +## When NOT to Use + +- Temporary conversation context that only matters in the current session +- Large file storage (MemWal is optimized for text memories) +- Use cases that don't need encryption or decentralization + +--- + +## Installation + +```bash +# Install the SDK +pnpm add @mysten-incubation/memwal + +# Optional: for Vercel AI SDK integration +pnpm add ai zod + +# Optional: for manual client (client-side SEAL encryption) +pnpm add @mysten/sui @mysten/seal @mysten/walrus +``` + +--- + +## Quick Start + +### 1. Get Your Credentials + +You need a **delegate key** (Ed25519 private key) and **account ID** (MemWalAccount object ID on Sui). + +Generate them at: +- Production: https://memwal.ai or https://memwal.wal.app +- Staging: https://staging.memwal.ai + +### 2. Initialize the SDK + +```ts +import { MemWal } from "@mysten-incubation/memwal"; + +const memwal = MemWal.create({ + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "my-app", +}); +``` + +### 3. Store and Recall Memories + +```ts +// Store a memory +await memwal.remember("User prefers dark mode and works in TypeScript."); + +// Recall by meaning +const result = await memwal.recall("What are the user's preferences?"); +console.log(result.results); + +// Extract and store facts from text +await memwal.analyze("I live in Hanoi and prefer dark mode."); + +// Check relayer health +await memwal.health(); +``` + +--- + +## SDK Entry Points + +| Entry Point | Import | Description | +|---|---|---| +| `MemWal` | `@mysten-incubation/memwal` | **Default.** Relayer handles embedding, SEAL encryption, Walrus upload, vector search | +| `MemWalManual` | `@mysten-incubation/memwal/manual` | Manual flow — client handles embedding and SEAL encryption | +| `withMemWal` | `@mysten-incubation/memwal/ai` | Vercel AI SDK middleware — auto recall + save around AI conversations | +| Account utils | `@mysten-incubation/memwal/account` | Account creation, delegate key management | + +--- + +## API Surface + +### MemWal Methods + +| Method | Description | Returns | +|---|---|---| +| `remember(text, namespace?)` | Store one memory (relayer embeds, encrypts, uploads) | `{ id, blob_id, owner, namespace }` | +| `recall(query, limit?, namespace?)` | Semantic search for memories | `{ results: [{ blob_id, text, distance }], total }` | +| `analyze(text, namespace?)` | Extract facts via LLM, store each as a memory | `{ facts: [{ text, id, blob_id }], total, owner }` | +| `restore(namespace, limit?)` | Rebuild missing index entries from Walrus | `{ restored, skipped, total, namespace, owner }` | +| `health()` | Check relayer health | `{ status, version }` | +| `getPublicKeyHex()` | Get hex-encoded public key | `string` | + +### Lower-Level Methods + +| Method | Description | +|---|---| +| `rememberManual({ blobId, vector, namespace? })` | Register pre-uploaded blob with pre-computed vector | +| `recallManual({ vector, limit?, namespace? })` | Search with pre-computed vector (returns blob IDs only) | +| `embed(text)` | Generate embedding vector (no storage) | + +--- + +## Configuration + +### MemWalConfig + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `key` | `string` | Yes | — | Ed25519 delegate private key in hex | +| `accountId` | `string` | Yes | — | MemWalAccount object ID on Sui | +| `serverUrl` | `string` | No | `http://localhost:8000` | Relayer URL | +| `namespace` | `string` | No | `"default"` | Default namespace for memory isolation | + +### Managed Relayer Endpoints + +| Network | Relayer URL | +|---|---| +| **Production** (mainnet) | `https://relayer.memwal.ai` | +| **Staging** (testnet) | `https://relayer.staging.memwal.ai` | + +--- + +## Vercel AI SDK Integration + +```ts +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { withMemWal } from "@mysten-incubation/memwal/ai"; + +const model = withMemWal(openai("gpt-4o"), { + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "chat", + maxMemories: 5, + autoSave: true, + minRelevance: 0.3, +}); + +const result = streamText({ + model, + messages: [{ role: "user", content: "What do you remember about me?" }], +}); +``` + +The middleware automatically: +- Recalls relevant memories before generation +- Extracts and saves facts from conversations after generation + +--- + +## OpenClaw / NemoClaw Plugin + +For OpenClaw agent integration, use the `@mysten-incubation/oc-memwal` plugin. + +### Install + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +### Configure + +Add to `~/.openclaw/openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "oc-memwal" }, + "entries": { + "oc-memwal": { + "enabled": true, + "config": { + "privateKey": "${MEMWAL_PRIVATE_KEY}", + "accountId": "0x...", + "serverUrl": "https://relayer.memwal.ai" + } + } + } + } +} +``` + +Lifecycle hooks run automatically: +- `before_prompt_build` — injects relevant memories as context +- `before_reset` — saves session summary +- `agent_end` — captures last response + +--- + +## Troubleshooting + +| Symptom | Fix | +|---|---| +| `health()` returns error | Check relayer URL is correct and reachable | +| `recall()` returns empty | Verify namespace matches what was used in `remember()` | +| `401 Unauthorized` | Verify delegate key is correct and registered on the account | +| SDK import errors | Run `pnpm add @mysten-incubation/memwal` — check Node.js ≥ 18 | +| Manual client errors | Install peer deps: `@mysten/sui @mysten/seal @mysten/walrus` | + +--- + +## Links + +- **Docs**: https://docs.memwal.ai +- **SDK on npm**: https://www.npmjs.com/package/@mysten-incubation/memwal +- **GitHub**: https://github.com/CommandOSSLabs/MemWal +- **Dashboard**: https://memwal.ai +- **llms.txt**: https://docs.memwal.ai/llms.txt diff --git a/docs/contributing/run-repo-locally.md b/docs/contributing/run-repo-locally.md index 7d5bae4e..ce3b0773 100644 --- a/docs/contributing/run-repo-locally.md +++ b/docs/contributing/run-repo-locally.md @@ -1,40 +1,131 @@ --- title: "Run the Repo Locally" +description: "Step-by-step guide to set up the MemWal monorepo for local development." --- -This monorepo contains: +## Prerequisites -- TypeScript applications under `apps/` -- the SDK under `packages/sdk` -- Rust backend services under `services/` -- Mintlify docs under `docs/` +| Tool | Version | Check | +|------|---------|-------| +| **Node.js** | ≥ 20 | `node -v` | +| **pnpm** | ≥ 9.12 | `pnpm -v` | +| **Rust** | latest stable (only for backend services) | `rustc --version` | -## Common Local Entry Points + +If you only work on TypeScript apps or docs, you don't need Rust. + -From the repository root: +## Step 1 — Clone and Install ```bash +git clone https://github.com/CommandOSSLabs/MemWal.git +cd MemWal pnpm install +``` + +## Step 2 — Build the SDK First + + +The apps depend on the SDK's compiled output. If you skip this step, apps will fail to start with import errors. + + +```bash +pnpm build:sdk +``` + +This compiles `packages/sdk` → `packages/sdk/dist/`. The apps import from `@mysten-incubation/memwal`, which resolves to this compiled output via the workspace. + +## Step 3 — Run What You Need + +Run individual surfaces from the repository root: + +```bash +# Docs site (Mintlify) pnpm dev:docs -pnpm dev:app -pnpm dev:noter -pnpm dev:chatbot -pnpm dev:researcher + +# Demo apps (pick one) +pnpm dev:app # Playground dashboard +pnpm dev:noter # Note-taking app +pnpm dev:chatbot # AI chatbot +pnpm dev:researcher # Research assistant + +# SDK in watch mode (recompiles on changes) +pnpm dev:sdk ``` -Backend services are run from their respective Rust service directories. +## Step 4 — Backend Services (Optional) -## Service Dependencies +The TypeScript apps talk to a managed relayer by default. You only need to run backend services if you're working on the relayer or indexer. -For relayer-oriented local work you will typically need: +### Relayer (`services/server`) -- PostgreSQL +Requires: +- PostgreSQL with `pgvector` extension - Sui RPC access - Walrus endpoints -- embedding provider credentials +- Embedding provider credentials (OpenAI-compatible) + +Quick start: + +```bash +# Start PostgreSQL with pgvector +docker compose -f services/server/docker-compose.yml up -d postgres + +# Configure environment +cp services/server/.env.example services/server/.env +# Edit .env with your credentials + +# Install sidecar dependencies +cd services/server/scripts && npm ci && cd .. + +# Run the relayer +cargo run +``` + +For the full relayer setup guide, see [Self-Hosting](/relayer/self-hosting). + +### Indexer (`services/indexer`) + +```bash +cd services/indexer +cargo run +``` + +The indexer polls Sui events and syncs account data into PostgreSQL. + +## Monorepo Structure + +``` +MemWal/ +├── packages/ +│ ├── sdk/ # @mysten-incubation/memwal — TypeScript SDK +│ └── openclaw-memory-memwal/ # @mysten-incubation/oc-memwal — OpenClaw plugin +├── apps/ +│ ├── app/ # Playground dashboard +│ ├── chatbot/ # AI chatbot demo +│ ├── noter/ # Note-taking demo +│ └── researcher/ # Research assistant demo +├── services/ +│ ├── server/ # Rust relayer (Axum) +│ ├── indexer/ # Rust Sui event indexer +│ └── contract/ # Move smart contract +├── docs/ # Mintlify documentation site +└── SKILL.md # Agent-first integration guide +``` + +## Troubleshooting -## Relayer Setup +| Problem | Cause | Fix | +|---------|-------|-----| +| `Cannot find module '@mysten-incubation/memwal'` | SDK not built | Run `pnpm build:sdk` first | +| `ERR_MODULE_NOT_FOUND` in apps | Stale SDK build | Run `pnpm build:sdk` again | +| `pnpm install` fails | Wrong pnpm version | Use pnpm ≥ 9.12: `corepack enable && corepack prepare pnpm@9.12.3 --activate` | +| Docs site won't start | Missing Mintlify | Run `pnpm install` from the root | +| Relayer crashes on boot | Missing pgvector | Install the `pgvector` PostgreSQL extension | +| Sidecar timeout | Missing sidecar deps | Run `cd services/server/scripts && npm ci` | -If you want to run the backend locally, start with the Relayer docs: +## See Also -- [Self-Hosting](/relayer/self-hosting) +- [Run Docs Locally](/contributing/run-docs-locally) — just the docs site +- [Self-Hosting](/relayer/self-hosting) — full relayer deployment +- [Environment Variables](/reference/environment-variables) — relayer configuration diff --git a/docs/docs.json b/docs/docs.json index a9c1044a..aed1fa5c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -84,7 +84,8 @@ "sdk/usage/with-memwal" ] }, - "sdk/api-reference" + "sdk/api-reference", + "sdk/changelog" ] } ] @@ -138,7 +139,8 @@ "openclaw/overview", "openclaw/quick-start", "openclaw/how-it-works", - "openclaw/reference" + "openclaw/reference", + "openclaw/changelog" ] } ] diff --git a/docs/llms-full.txt b/docs/llms-full.txt new file mode 100644 index 00000000..89a78093 --- /dev/null +++ b/docs/llms-full.txt @@ -0,0 +1,452 @@ +# MemWal + +> MemWal is a privacy-first AI memory layer. It stores encrypted memories on Walrus (decentralized storage) and retrieves them with semantic search. Ownership is enforced onchain via Sui smart contracts. The TypeScript SDK (`@mysten-incubation/memwal`) gives any app persistent, encrypted memory in a few lines of code. + +Important notes: + +- MemWal is currently in beta +- The SDK talks to a relayer service that handles embedding, SEAL encryption, Walrus upload/download, and vector search +- All content is end-to-end encrypted — only the owner and authorized delegates can decrypt +- Delegate keys provide scoped access — agents can read/write memory without holding the owner's private key +- Memory is scoped by `owner + namespace` — each namespace is an isolated memory space + +--- + +## Quick Start + +### Installation + +```bash +pnpm add @mysten-incubation/memwal +``` + +Optional peer dependencies: + +```bash +# For Vercel AI SDK middleware +pnpm add ai zod + +# For manual client (client-side SEAL encryption) +pnpm add @mysten/sui @mysten/seal @mysten/walrus +``` + +### Prerequisites + +- Node.js v18+ or Bun v1+ +- A delegate key and account ID (generate at https://memwal.ai or https://memwal.wal.app) +- A relayer URL (use `https://relayer.memwal.ai` for production or `https://relayer.staging.memwal.ai` for staging) + +### First Memory + +```ts +import { MemWal } from "@mysten-incubation/memwal"; + +const memwal = MemWal.create({ + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "demo", +}); + +await memwal.health(); +await memwal.remember("I live in Hanoi and prefer dark mode."); + +const result = await memwal.recall("What do we know about this user?"); +console.log(result.results); +``` + +--- + +## SDK Entry Points + +| Entry Point | Import | When to Use | +|---|---|---| +| `MemWal` | `@mysten-incubation/memwal` | **Recommended default** — relayer handles embeddings, SEAL, and storage | +| `MemWalManual` | `@mysten-incubation/memwal/manual` | Client-managed embeddings and local SEAL operations | +| `withMemWal` | `@mysten-incubation/memwal/ai` | Vercel AI SDK middleware — auto recall + save | +| Account utils | `@mysten-incubation/memwal/account` | Account creation, delegate key management | + +--- + +## SDK API Reference + +### `MemWal.create(config)` + +```ts +MemWal.create(config: MemWalConfig): MemWal +``` + +Config: + +| Property | Type | Required | Default | Notes | +|---|---|---|---|---| +| `key` | `string` | Yes | — | Ed25519 delegate private key in hex | +| `accountId` | `string` | Yes | — | MemWalAccount object ID on Sui | +| `serverUrl` | `string` | No | `http://localhost:8000` | Relayer URL | +| `namespace` | `string` | No | `"default"` | Default namespace for memory isolation | + +### `remember(text, namespace?): Promise` + +Store one memory through the relayer. The relayer handles embedding, SEAL encryption, Walrus upload, and vector indexing. + +Returns: +```ts +{ + id: string; // UUID for this entry + blob_id: string; // Walrus blob ID + owner: string; // Owner Sui address + namespace: string; // Namespace used +} +``` + +### `recall(query, limit?, namespace?): Promise` + +Search for memories matching a natural language query, scoped to `owner + namespace`. + +- `limit` defaults to `10` + +Returns: +```ts +{ + results: Array<{ + blob_id: string; // Walrus blob ID + text: string; // Decrypted plaintext + distance: number; // Cosine distance (lower = more similar) + }>; + total: number; +} +``` + +### `analyze(text, namespace?): Promise` + +Extract memorable facts from text using an LLM, then store each fact as a separate memory. + +Returns: +```ts +{ + facts: Array<{ + text: string; // Extracted fact + id: string; // UUID + blob_id: string; // Walrus blob ID + }>; + total: number; + owner: string; +} +``` + +### `restore(namespace, limit?): Promise` + +Rebuild missing indexed entries for one namespace from Walrus. Incremental — only re-indexes blobs that aren't already in the local database. + +- `limit` defaults to `50` + +Returns: +```ts +{ + restored: number; // Entries newly indexed + skipped: number; // Entries already in DB + total: number; // Total blobs found on-chain + namespace: string; + owner: string; +} +``` + +### `health(): Promise` + +Check relayer health. Does not require authentication. + +Returns: `{ status: string, version: string }` + +### `getPublicKeyHex(): Promise` + +Return the hex-encoded public key for the current delegate key. + +### Lower-level methods + +| Method | Description | +|--------|-------------| +| `rememberManual({ blobId, vector, namespace? })` | Register a pre-uploaded blob ID with a pre-computed vector | +| `recallManual({ vector, limit?, namespace? })` | Search with a pre-computed query vector (returns blob IDs, no decryption) | +| `embed(text)` | Generate an embedding vector for text (no storage) | + +--- + +## MemWalManual + +```ts +import { MemWalManual } from "@mysten-incubation/memwal/manual"; +``` + +Manual client flow — embed locally, SEAL encrypt locally, send encrypted payload + vector to relayer. + +### Config (extends MemWalConfig) + +| Field | Required | Notes | +|---|---|---| +| `embeddingApiKey` | Yes | OpenAI/OpenRouter-compatible embedding key | +| `embeddingApiBase` | No | Default: `https://api.openai.com/v1` | +| `embeddingModel` | No | Default: `text-embedding-3-small` | +| `packageId` | Yes | MemWal package ID on Sui | +| `suiPrivateKey` or `walletSigner` | One required | Local keypair or connected wallet | +| `suiNetwork` | No | Default: `mainnet` | + +--- + +## withMemWal (AI Middleware) + +```ts +import { withMemWal } from "@mysten-incubation/memwal/ai"; +``` + +Wraps a Vercel AI SDK model with automatic memory recall and save. + +```ts +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { withMemWal } from "@mysten-incubation/memwal/ai"; + +const model = withMemWal(openai("gpt-4o"), { + key: "", + accountId: "", + serverUrl: "https://relayer.memwal.ai", + namespace: "chat", + maxMemories: 5, + autoSave: true, + minRelevance: 0.3, +}); + +const result = streamText({ + model, + messages: [{ role: "user", content: "What do you remember about me?" }], +}); +``` + +**Before generation:** +- Reads the last user message +- Runs `recall()` against MemWal +- Filters by minimum relevance (`minRelevance`, default `0.3`) +- Injects matching memories into the prompt as a system message + +**After generation:** +- Optionally runs `analyze()` on the user message (fire-and-forget) +- Saves extracted facts asynchronously + +Options (extends MemWalConfig): + +| Option | Default | Description | +|--------|---------|-------------| +| `maxMemories` | `5` | Max memories to inject per request | +| `autoSave` | `true` | Auto-save new facts from conversation | +| `minRelevance` | `0.3` | Minimum similarity score (0–1) to include a memory | +| `debug` | `false` | Enable debug logging | + +--- + +## Account Management + +```ts +import { + createAccount, + addDelegateKey, + removeDelegateKey, + generateDelegateKey, +} from "@mysten-incubation/memwal/account"; +``` + +| Function | Description | +|----------|-------------| +| `generateDelegateKey()` | Generate a new Ed25519 keypair (returns `privateKey`, `publicKey`, `suiAddress`) | +| `createAccount(opts)` | Create a new MemWalAccount on-chain (one per Sui address) | +| `addDelegateKey(opts)` | Add a delegate key to an account (owner only) | +| `removeDelegateKey(opts)` | Remove a delegate key from an account (owner only) | + +--- + +## Utility Functions + +```ts +import { delegateKeyToSuiAddress, delegateKeyToPublicKey } from "@mysten-incubation/memwal"; +``` + +| Function | Description | +|----------|-------------| +| `delegateKeyToSuiAddress(privateKeyHex)` | Derive the Sui address from a delegate private key | +| `delegateKeyToPublicKey(privateKeyHex)` | Get the 32-byte public key from a delegate private key | + +--- + +## Configuration Reference + +### MemWalConfig + +Used by `MemWal.create(config)` and `withMemWal(model, options)`. + +| Field | Required | Notes | +|---|---|---| +| `key` | yes | Delegate private key in hex | +| `accountId` | yes | MemWalAccount object ID on Sui | +| `serverUrl` | no | Relayer URL. Default: `http://localhost:8000` | +| `namespace` | no | Default memory boundary. Default: `"default"` | + +### MemWalManualConfig + +Used by `MemWalManual.create(config)`. + +| Field | Required | Notes | +|---|---|---| +| `key` | yes | Delegate private key in hex | +| `serverUrl` | no | Relayer URL | +| `embeddingApiKey` | yes | OpenAI/OpenRouter-compatible embedding key | +| `embeddingApiBase` | no | Default: `https://api.openai.com/v1` | +| `embeddingModel` | no | Default: `text-embedding-3-small` | +| `packageId` | yes | MemWal package ID on Sui | +| `accountId` | yes | MemWalAccount object ID | +| `namespace` | no | Default namespace | +| `suiPrivateKey` | one of two | Use for local signing | +| `walletSigner` | one of two | Use a connected browser wallet instead | +| `suiNetwork` | no | `testnet` or `mainnet`. Default: `mainnet` | + +### Managed Relayer Endpoints + +| Network | Relayer URL | +|---|---| +| **Production** (mainnet) | `https://relayer.memwal.ai` | +| **Staging** (testnet) | `https://relayer.staging.memwal.ai` | + +--- + +## Relayer Overview + +The relayer is the backend that turns SDK calls into memory operations: + +- **Authenticates requests** by verifying Ed25519 signatures against onchain delegate keys +- **Generates embeddings** using an OpenAI-compatible API (default: `text-embedding-3-small`, 1536 dimensions) +- **Encrypts and decrypts** data through the SEAL sidecar +- **Uploads and downloads** encrypted blobs to/from Walrus +- **Stores and searches vectors** in PostgreSQL (pgvector) +- **Orchestrates flows** like `analyze` (LLM fact extraction) and `ask` (memory-augmented Q&A) +- **Restores memory spaces** by querying onchain blobs, decrypting, and re-indexing + +### Authentication Headers + +| Header | Description | +|--------|-------------| +| `x-public-key` | Hex-encoded Ed25519 public key (32 bytes) | +| `x-signature` | Hex-encoded Ed25519 signature (64 bytes) | +| `x-timestamp` | Unix timestamp in seconds (5-minute validity window) | +| `x-account-id` | Optional — MemWalAccount object ID hint | + +Signature format: `{timestamp}.{method}.{path}.{body_sha256}` + +### Relayer API Routes + +| Method | Route | Description | +|--------|-------|-------------| +| `GET` | `/health` | Service health check (no auth) | +| `POST` | `/api/remember` | Store text as encrypted memory | +| `POST` | `/api/recall` | Semantic search for memories | +| `POST` | `/api/remember/manual` | Register client-encrypted payload | +| `POST` | `/api/recall/manual` | Search with pre-computed vector | +| `POST` | `/api/analyze` | Extract facts via LLM, store each | +| `POST` | `/api/ask` | Memory-augmented Q&A | +| `POST` | `/api/restore` | Rebuild missing index entries | + +--- + +## OpenClaw / NemoClaw Plugin + +### Installation + +```bash +openclaw plugins install @mysten-incubation/oc-memwal +``` + +### Configuration + +Set the delegate key as an environment variable: + +```bash +export MEMWAL_PRIVATE_KEY="your-64-char-hex-key" +``` + +Add to `~/.openclaw/openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "oc-memwal" }, + "entries": { + "oc-memwal": { + "enabled": true, + "config": { + "privateKey": "${MEMWAL_PRIVATE_KEY}", + "accountId": "0x...", + "serverUrl": "https://relayer.memwal.ai" + } + } + } + } +} +``` + +### Plugin Options + +| Option | Default | Description | +|--------|---------|-------------| +| `autoRecall` | `true` | Inject relevant memories before each turn | +| `autoCapture` | `true` | Extract and store facts after each turn | +| `maxRecallResults` | `5` | Max memories injected per turn | +| `minRelevance` | `0.3` | Relevance threshold (0-1) | +| `captureMaxMessages` | `10` | Messages to analyze for facts | +| `defaultNamespace` | `"default"` | Memory scope for the main agent | + +### Lifecycle Hooks + +| Hook | Trigger | What Happens | +|------|---------|--------------| +| `before_prompt_build` | Every LLM call | Relevant memories injected as context | +| `before_reset` | Before `/reset` | Session summary saved | +| `agent_end` | Agent finishes | Last response captured | + +--- + +## Environment Variables (Self-Hosting) + +### Required + +| Variable | Notes | +|---|---| +| `DATABASE_URL` | PostgreSQL connection string. `pgvector` must already exist | +| `MEMWAL_PACKAGE_ID` | Sui package ID | +| `MEMWAL_REGISTRY_ID` | Onchain registry object ID | +| `SEAL_KEY_SERVERS` | Comma-separated SEAL key server object IDs | + +### Usually Required + +| Variable | Notes | +|---|---| +| `SERVER_SUI_PRIVATE_KEY` | Primary server key | +| `OPENAI_API_KEY` | Embedding and fact-extraction provider | + +### Package Contract IDs + +Staging (Testnet): +``` +MEMWAL_PACKAGE_ID=0xcf6ad755a1cdff7217865c796778fabe5aa399cb0cf2eba986f4b582047229c6 +MEMWAL_REGISTRY_ID=0xe80f2feec1c139616a86c9f71210152e2a7ca552b20841f2e192f99f75864437 +``` + +Production (Mainnet): +``` +MEMWAL_PACKAGE_ID=0xcee7a6fd8de52ce645c38332bde23d4a30fd9426bc4681409733dd50958a24c6 +MEMWAL_REGISTRY_ID=0x0da982cefa26864ae834a8a0504b904233d49e20fcc17c373c8bed99c75a7edd +``` + +--- + +## Links + +- **Docs**: https://docs.memwal.ai +- **SDK on npm**: https://www.npmjs.com/package/@mysten-incubation/memwal +- **GitHub**: https://github.com/CommandOSSLabs/MemWal +- **Dashboard**: https://memwal.ai diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 00000000..e7bcd343 --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,58 @@ +# MemWal + +> MemWal is a privacy-first AI memory layer. It stores encrypted memories on Walrus (decentralized storage) and retrieves them with semantic search. Ownership is enforced onchain via Sui smart contracts. The TypeScript SDK (`@mysten-incubation/memwal`) gives any app persistent, encrypted memory in a few lines of code. + +Important notes: + +- MemWal is currently in beta +- The SDK talks to a relayer service that handles embedding, SEAL encryption, Walrus upload/download, and vector search +- All content is end-to-end encrypted — only the owner and authorized delegates can decrypt +- Delegate keys provide scoped access — agents can read/write memory without holding the owner's private key +- Memory is scoped by `owner + namespace` — each namespace is an isolated memory space + +## Docs + +- [What is MemWal?](https://docs.memwal.ai/getting-started/what-is-memwal): Overview of features, architecture, and use cases +- [Quick Start](https://docs.memwal.ai/getting-started/quick-start): Install the SDK, generate credentials, store and recall your first memory +- [Choose Your Path](https://docs.memwal.ai/getting-started/choose-your-path): Pick the right integration path (MemWal, MemWalManual, or withMemWal AI middleware) + +## SDK + +- [SDK Quick Start](https://docs.memwal.ai/sdk/quick-start): Install, configure, and store your first memory +- [MemWal Usage](https://docs.memwal.ai/sdk/usage/memwal): Default relayer-handled client — remember, recall, analyze, restore +- [MemWalManual Usage](https://docs.memwal.ai/sdk/usage/memwal-manual): Manual client — client-side embedding and SEAL encryption +- [withMemWal AI Middleware](https://docs.memwal.ai/sdk/usage/with-memwal): Vercel AI SDK integration — automatic memory recall and save +- [SDK API Reference](https://docs.memwal.ai/sdk/api-reference): Full method signatures, return types, and config fields + +## Relayer + +- [Relayer Overview](https://docs.memwal.ai/relayer/overview): Architecture, trust boundary, key pool, single-instance design +- [Public Relayer](https://docs.memwal.ai/relayer/public-relayer): Managed relayer endpoints provided by Walrus Foundation +- [Self-Hosting](https://docs.memwal.ai/relayer/self-hosting): Run your own relayer with full control over encryption and embedding +- [Relayer API Reference](https://docs.memwal.ai/relayer/api-reference): HTTP routes, authentication headers, request/response shapes + +## Smart Contract + +- [Contract Overview](https://docs.memwal.ai/contract/overview): Onchain ownership model and package IDs +- [Delegate Key Management](https://docs.memwal.ai/contract/delegate-key-management): Add, remove, and rotate delegate keys +- [Ownership & Permissions](https://docs.memwal.ai/contract/ownership-and-permissions): Access control model + +## OpenClaw Plugin + +- [OpenClaw Overview](https://docs.memwal.ai/openclaw/overview): NemoClaw/OpenClaw memory plugin overview +- [OpenClaw Quick Start](https://docs.memwal.ai/openclaw/quick-start): Install and configure the plugin +- [How It Works](https://docs.memwal.ai/openclaw/how-it-works): Architecture, hooks, and message flow +- [OpenClaw Reference](https://docs.memwal.ai/openclaw/reference): Hooks, tools, CLI, and configuration + +## Reference + +- [Configuration](https://docs.memwal.ai/reference/configuration): MemWalConfig, MemWalManualConfig, WithMemWalOptions +- [Environment Variables](https://docs.memwal.ai/reference/environment-variables): Relayer env vars for self-hosting + +## Optional + +- [Concepts: Memory Space](https://docs.memwal.ai/fundamentals/concepts/memory-space): Namespace isolation and memory boundaries +- [Architecture: Core Components](https://docs.memwal.ai/fundamentals/architecture/core-components): System overview and component responsibilities +- [Data Flow & Security Model](https://docs.memwal.ai/fundamentals/architecture/data-flow-security-model): Trust boundaries and encryption flows +- [Contributing: Run Repo Locally](https://docs.memwal.ai/contributing/run-repo-locally): Monorepo setup for contributors +- [Example Apps](https://docs.memwal.ai/examples/example-apps): Playground, Chatbot, Noter, Researcher diff --git a/docs/openclaw/changelog.md b/docs/openclaw/changelog.md new file mode 100644 index 00000000..1e3a3ce3 --- /dev/null +++ b/docs/openclaw/changelog.md @@ -0,0 +1,19 @@ +--- +title: "Changelog" +description: "Release history for the MemWal OpenClaw plugin." +--- + +Track what's new, changed, and fixed in `@mysten-incubation/oc-memwal`. + +For the latest version, see the [npm package page](https://www.npmjs.com/package/@mysten-incubation/oc-memwal). + +## 0.0.1 + +### Initial Release + +- NemoClaw/OpenClaw memory plugin powered by MemWal +- Automatic memory recall via `before_prompt_build` hook +- Automatic fact capture via `agent_end` hook +- Session summary on `before_reset` hook +- CLI commands: `openclaw memwal stats`, `openclaw memwal search` +- LLM tools: `memory_search`, `memory_store` diff --git a/docs/sdk/changelog.md b/docs/sdk/changelog.md new file mode 100644 index 00000000..b651ff12 --- /dev/null +++ b/docs/sdk/changelog.md @@ -0,0 +1,19 @@ +--- +title: "Changelog" +description: "Release history for the MemWal TypeScript SDK." +--- + +Track what's new, changed, and fixed in `@mysten-incubation/memwal`. + +For the latest version, see the [npm package page](https://www.npmjs.com/package/@mysten-incubation/memwal). + +## 0.0.1 + +### Initial Release + +- `MemWal` default client — relayer-handled embedding, SEAL encryption, Walrus upload, vector search +- `MemWalManual` manual client — client-side embedding and SEAL operations +- `withMemWal` Vercel AI SDK middleware — automatic memory recall and save +- Account management utilities — `createAccount`, `addDelegateKey`, `removeDelegateKey`, `generateDelegateKey` +- Ed25519 delegate key authentication +- Namespace-scoped memory isolation diff --git a/packages/openclaw-memory-memwal/CHANGELOG.md b/packages/openclaw-memory-memwal/CHANGELOG.md new file mode 100644 index 00000000..c42108de --- /dev/null +++ b/packages/openclaw-memory-memwal/CHANGELOG.md @@ -0,0 +1,12 @@ +# @mysten-incubation/oc-memwal + +## 0.0.1 + +### Initial Release + +- NemoClaw/OpenClaw memory plugin powered by MemWal +- Automatic memory recall via `before_prompt_build` hook +- Automatic fact capture via `agent_end` hook +- Session summary on `before_reset` hook +- CLI commands: `openclaw memwal stats`, `openclaw memwal search` +- LLM tools: `memory_search`, `memory_store` diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md new file mode 100644 index 00000000..ea0c0192 --- /dev/null +++ b/packages/sdk/CHANGELOG.md @@ -0,0 +1,12 @@ +# @mysten-incubation/memwal + +## 0.0.1 + +### Initial Release + +- `MemWal` default client — relayer-handled embedding, SEAL encryption, Walrus upload, vector search +- `MemWalManual` manual client — client-side embedding and SEAL operations +- `withMemWal` Vercel AI SDK middleware — automatic memory recall and save +- Account management utilities — `createAccount`, `addDelegateKey`, `removeDelegateKey`, `generateDelegateKey` +- Ed25519 delegate key authentication +- Namespace-scoped memory isolation From 0fd01e5944420839afeec9598b8b3ddd6685b7c7 Mon Sep 17 00:00:00 2001 From: ducnmm Date: Tue, 31 Mar 2026 14:35:34 +0700 Subject: [PATCH 07/10] fix: enforce --frozen-lockfile in all CI workflows and Dockerfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - deploy-app-walrus.yml: --no-frozen-lockfile → --frozen-lockfile - release-oc-memwal.yml: --no-frozen-lockfile → --frozen-lockfile - release-sdk.yml: --no-frozen-lockfile → --frozen-lockfile - apps/app/Dockerfile: pnpm install → pnpm install --frozen-lockfile - apps/researcher/Dockerfile: bun install → bun install --frozen-lockfile - apps/researcher/package.json: pin @mysten-incubation/memwal to exact 0.0.1 Prevents supply chain attacks like axios@1.14.1 compromise (axios/axios#10604) from being silently pulled into CI builds. --- .github/workflows/deploy-app-walrus.yml | 2 +- .github/workflows/release-oc-memwal.yml | 2 +- .github/workflows/release-sdk.yml | 2 +- apps/app/Dockerfile | 2 +- apps/researcher/Dockerfile | 2 +- apps/researcher/package.json | 2 +- pnpm-lock.yaml | 69 ++++++++++++++++++++++--- 7 files changed, 68 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-app-walrus.yml b/.github/workflows/deploy-app-walrus.yml index 29062c60..60d72aaa 100644 --- a/.github/workflows/deploy-app-walrus.yml +++ b/.github/workflows/deploy-app-walrus.yml @@ -30,7 +30,7 @@ jobs: uses: pnpm/action-setup@v4 - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Validate ws-resources.json exists run: | diff --git a/.github/workflows/release-oc-memwal.yml b/.github/workflows/release-oc-memwal.yml index 9994accc..298452a3 100644 --- a/.github/workflows/release-oc-memwal.yml +++ b/.github/workflows/release-oc-memwal.yml @@ -40,7 +40,7 @@ jobs: run: npm install -g npm@latest - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Build SDK (dependency) run: pnpm build:sdk diff --git a/.github/workflows/release-sdk.yml b/.github/workflows/release-sdk.yml index 401414f5..305d2ae5 100644 --- a/.github/workflows/release-sdk.yml +++ b/.github/workflows/release-sdk.yml @@ -40,7 +40,7 @@ jobs: run: npm install -g npm@latest - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: pnpm install --frozen-lockfile - name: Typecheck run: pnpm --filter @mysten-incubation/memwal typecheck diff --git a/apps/app/Dockerfile b/apps/app/Dockerfile index 593499d0..6b19f37d 100644 --- a/apps/app/Dockerfile +++ b/apps/app/Dockerfile @@ -18,7 +18,7 @@ COPY packages/sdk/package.json ./packages/sdk/ COPY apps/app/package.json ./apps/app/ # Install all workspace deps -RUN pnpm install +RUN pnpm install --frozen-lockfile # Build SDK first (apps/app depends on @mysten-incubation/memwal via workspace:*) COPY packages/sdk/ ./packages/sdk/ diff --git a/apps/researcher/Dockerfile b/apps/researcher/Dockerfile index 221ea3fd..9640bc8d 100644 --- a/apps/researcher/Dockerfile +++ b/apps/researcher/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /app COPY apps/researcher/package.json ./ # Install deps — @mysten-incubation/memwal is now on npm, no local SDK needed -RUN bun install +RUN bun install --frozen-lockfile # ── Stage 2: Build ───────────────────────────────────────── FROM oven/bun:1 AS builder diff --git a/apps/researcher/package.json b/apps/researcher/package.json index 0217aa35..517ebec7 100644 --- a/apps/researcher/package.json +++ b/apps/researcher/package.json @@ -24,7 +24,7 @@ "@ai-sdk/openai": "^3.0.41", "@ai-sdk/provider": "^3.0.3", "@ai-sdk/react": "3.0.39", - "@mysten-incubation/memwal": "^0.0.1-dev.0", + "@mysten-incubation/memwal": "0.0.1", "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", "@codemirror/state": "^6.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23f0458a..ae6e5569 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -702,8 +702,8 @@ importers: specifier: ^13.7.0 version: 13.12.0(react@19.0.1) '@mysten-incubation/memwal': - specifier: workspace:* - version: link:../../packages/sdk + specifier: 0.0.1 + version: 0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76) '@noble/ed25519': specifier: ^2.2.3 version: 2.3.0 @@ -973,6 +973,25 @@ importers: specifier: ^4 version: 4.2.441(@radix-ui/react-popover@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/node@24.12.0)(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(typescript@5.9.3) + packages/openclaw-memory-memwal: + dependencies: + '@mysten-incubation/memwal': + specifier: ^0.0.1 + version: 0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76) + '@sinclair/typebox': + specifier: 0.34.48 + version: 0.34.48 + zod: + specifier: ^3.23.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.19.15 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + packages/sdk: dependencies: '@mysten/seal': @@ -3045,6 +3064,26 @@ packages: resolution: {integrity: sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==} engines: {node: '>=18'} + '@mysten-incubation/memwal@0.0.1': + resolution: {integrity: sha512-kRAFFJBdk3D9XvGHZdPOrnz2x4C7dwCRf0xTaeLFAVTgVwfpk3GmOnJZ1O+pAQyrAhweAzBXNXBWutShnPWgJg==} + peerDependencies: + '@mysten/seal': '>=1.1.0' + '@mysten/sui': '>=2.5.0' + '@mysten/walrus': '>=1.0.3' + ai: '>=4.0.0' + zod: ^3.23.0 + peerDependenciesMeta: + '@mysten/seal': + optional: true + '@mysten/sui': + optional: true + '@mysten/walrus': + optional: true + ai: + optional: true + zod: + optional: true + '@mysten/bcs@1.2.0': resolution: {integrity: sha512-LuKonrGdGW7dq/EM6U2L9/as7dFwnhZnsnINzB/vu08Xfrj0qzWwpLOiXagAa5yZOPLK7anRZydMonczFkUPzA==} @@ -4458,6 +4497,9 @@ packages: '@shikijs/vscode-textmate@10.0.2': resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + '@sinclair/typebox@0.34.48': + resolution: {integrity: sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==} + '@sindresorhus/is@5.6.0': resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} engines: {node: '>=14.16'} @@ -13486,6 +13528,17 @@ snapshots: outvariant: 1.4.3 strict-event-emitter: 0.5.1 + '@mysten-incubation/memwal@0.0.1(@mysten/seal@1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)))(@mysten/sui@2.8.0(typescript@5.9.3))(@mysten/walrus@1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)))(ai@6.0.37(zod@3.25.76))(zod@3.25.76)': + dependencies: + '@noble/ed25519': 2.3.0 + '@noble/hashes': 2.0.1 + optionalDependencies: + '@mysten/seal': 1.1.1(@mysten/sui@2.8.0(typescript@5.9.3)) + '@mysten/sui': 2.8.0(typescript@5.9.3) + '@mysten/walrus': 1.0.4(@mysten/sui@2.8.0(typescript@5.9.3)) + ai: 6.0.37(zod@3.25.76) + zod: 3.25.76 + '@mysten/bcs@1.2.0': dependencies: bs58: 6.0.0 @@ -16528,6 +16581,8 @@ snapshots: '@shikijs/vscode-textmate@10.0.2': {} + '@sinclair/typebox@0.34.48': {} + '@sindresorhus/is@5.6.0': {} '@sindresorhus/merge-streams@4.0.0': {} @@ -16909,7 +16964,7 @@ snapshots: '@types/cors@2.8.19': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 '@types/d3-array@3.2.2': {} @@ -16958,7 +17013,7 @@ snapshots: '@types/es-aggregate-error@1.0.6': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 '@types/estree-jsx@1.0.5': dependencies: @@ -17088,7 +17143,7 @@ snapshots: '@types/yauzl@2.10.3': dependencies: - '@types/node': 20.19.37 + '@types/node': 22.19.15 optional: true '@typescript-eslint/eslint-plugin@8.57.0(@typescript-eslint/parser@8.57.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3)': @@ -18646,7 +18701,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.19 - '@types/node': 20.19.37 + '@types/node': 22.19.15 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -22163,7 +22218,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.19.37 + '@types/node': 22.19.15 long: 5.3.2 proxy-addr@2.0.7: From 41c295c05c19d60572130f738b879865dd2c6d3d Mon Sep 17 00:00:00 2001 From: ducnmm Date: Tue, 31 Mar 2026 22:31:19 +0700 Subject: [PATCH 08/10] fix(server): sponsor walrus register flow and use pooled signer keys --- services/server/scripts/sidecar-server.ts | 101 +++++++++++++++++++--- services/server/src/routes.rs | 18 ++-- 2 files changed, 97 insertions(+), 22 deletions(-) diff --git a/services/server/scripts/sidecar-server.ts b/services/server/scripts/sidecar-server.ts index 1d35a487..ca22dd8b 100644 --- a/services/server/scripts/sidecar-server.ts +++ b/services/server/scripts/sidecar-server.ts @@ -76,6 +76,49 @@ const walrusClient = new WalrusClient({ }, }); +const COIN_WITH_BALANCE_INTENT = "CoinWithBalance"; +const GAS_INTENT_TYPE = "gas"; +const SUI_TYPE = "0x2::sui::SUI"; +type TxIntentCommand = { + $kind?: string; + $Intent?: { + name?: string; + data?: { type?: string }; + }; +}; +type TxDataWithCommands = { commands: TxIntentCommand[] }; +type UploadRelayTipConfigResponse = { + send_tip?: { + address?: string; + }; +}; + +/** + * Rewrite CoinWithBalance "gas" intents to explicit SUI coin type so Enoki + * sponsorship can build the transaction (Enoki rejects GasCoin tx arguments). + */ +function patchGasCoinIntents(tx: Transaction): void { + tx.addSerializationPlugin(async (transactionData: TxDataWithCommands, _buildOptions, next) => { + let patched = 0; + for (const command of transactionData.commands) { + if ( + command.$kind === "$Intent" && + command.$Intent?.name === COIN_WITH_BALANCE_INTENT && + command.$Intent?.data?.type === GAS_INTENT_TYPE + ) { + command.$Intent.data.type = SUI_TYPE; + patched += 1; + } + } + + if (patched > 0) { + console.log(`[patch] converted ${patched} CoinWithBalance intent(s) from GasCoin -> sender SUI coins`); + } + + await next(); + }); +} + const ENOKI_API_BASE_URL = "https://api.enoki.mystenlabs.com/v1"; const enokiApiKey = process.env.ENOKI_API_KEY; const enokiNetwork = (process.env.ENOKI_NETWORK || process.env.SUI_NETWORK || "mainnet") as @@ -87,6 +130,38 @@ type EnokiDataWrapper = { data: T }; type EnokiSponsorResponse = { bytes: string; digest: string }; type EnokiExecuteResponse = { digest: string }; const signerUploadQueues = new Map>(); +let uploadRelayTipAddressCache: string | null | undefined = undefined; + +function dedupeAddresses(addresses: (string | null | undefined)[]): string[] { + return [...new Set(addresses.filter((addr): addr is string => typeof addr === "string" && addr.length > 0))]; +} + +async function getUploadRelayTipAddress(): Promise { + if (uploadRelayTipAddressCache !== undefined) { + return uploadRelayTipAddressCache; + } + + try { + const resp = await fetch(`${WALRUS_UPLOAD_RELAY_URL}/v1/tip-config`); + if (!resp.ok) { + throw new Error(`tip-config request failed (${resp.status})`); + } + + const json = await resp.json() as UploadRelayTipConfigResponse; + const address = json.send_tip?.address; + if (typeof address === "string" && address.startsWith("0x")) { + uploadRelayTipAddressCache = address; + return address; + } + + uploadRelayTipAddressCache = null; + return null; + } catch (err: any) { + console.warn(`[upload-relay] could not load tip-config: ${err.message || err}`); + // Don't cache transient failures; retry on next request. + return null; + } +} async function callEnoki(path: string, payload: unknown): Promise { if (!enokiApiKey) { @@ -147,12 +222,8 @@ async function executeWithEnokiSponsor(tx: Transaction, signer: Ed25519Keypair, return executed.digest; } catch (err: any) { - console.warn(`[enoki-sponsor] sponsor failed, falling back to direct signing: ${err.message}`); - const direct = await suiClient.signAndExecuteTransaction({ - signer, - transaction: tx, - }); - return direct.digest; + console.error(`[enoki-sponsor] sponsor failed: ${err.message}`); + throw err; } } @@ -454,12 +525,16 @@ app.post("/walrus/upload", async (req, res) => { }, }); - // Wait until register tx is confirmed before starting upload/certify. - // NOTE: Walrus register uses GasCoin internally — cannot be Enoki-sponsored - const registerResult = await suiClient.signAndExecuteTransaction({ signer, transaction: registerTx }); - await suiClient.waitForTransaction({ digest: registerResult.digest }); + // Patch: convert GasCoin intents → sender's SUI coins. + // Enoki rejects GasCoin as tx argument, but relay requires the tip. + // After patching, signer pays tip from own SUI; Enoki sponsors gas. + patchGasCoinIntents(registerTx); + const tipRecipient = await getUploadRelayTipAddress(); + const registerAllowedAddresses = dedupeAddresses([signerAddress, tipRecipient]); + const registerDigest = await executeWithEnokiSponsor(registerTx, signer, registerAllowedAddresses); + await suiClient.waitForTransaction({ digest: registerDigest }); - await flow.upload({ digest: registerResult.digest }); + await flow.upload({ digest: registerDigest }); const certifyTx = flow.certify(); // Wait until certify tx is confirmed before returning this upload. @@ -525,9 +600,9 @@ app.post("/walrus/upload", async (req, res) => { // Transfer blob to user metaTx.transferObjects([blobArg], owner); - const metaDigest = await executeWithEnokiSponsor(metaTx, signer, [owner]); + const metaDigest = await executeWithEnokiSponsor(metaTx, signer, dedupeAddresses([signerAddress, owner])); await suiClient.waitForTransaction({ digest: metaDigest }); - console.error(`[walrus/upload] metadata set + transferred blob ${blobObjectId} to ${owner} (ns=${namespace})`); + console.log(`[walrus/upload] metadata set + transferred blob ${blobObjectId} to ${owner} (ns=${namespace})`); } catch (metaErr: any) { // Non-fatal: blob is uploaded but metadata/transfer failed console.error(`[walrus/upload] metadata+transfer failed: ${metaErr.message}`); diff --git a/services/server/src/routes.rs b/services/server/src/routes.rs index 8d52b865..8c30a300 100644 --- a/services/server/src/routes.rs +++ b/services/server/src/routes.rs @@ -150,12 +150,12 @@ pub async fn remember( let encrypted = encrypted_result?; // Step 2: Upload encrypted blob → Walrus (via sidecar) - let sui_key = state.config.sui_private_key.as_deref().ok_or_else(|| { - AppError::Internal("SERVER_SUI_PRIVATE_KEY required for Walrus upload".into()) - })?; + let sui_key = state.key_pool.next() + .map(|s| s.to_string()) + .ok_or_else(|| AppError::Internal("No Sui keys configured (set SERVER_SUI_PRIVATE_KEYS or SERVER_SUI_PRIVATE_KEY)".into()))?; let upload_result = walrus::upload_blob( &state.http_client, &state.config.sidecar_url, - &encrypted, 50, owner, sui_key, namespace, &state.config.package_id, + &encrypted, 50, owner, &sui_key, namespace, &state.config.package_id, ).await?; let blob_id = upload_result.blob_id; @@ -313,10 +313,10 @@ pub async fn remember_manual( // Check storage quota before upload rate_limit::check_storage_quota(&state, owner, encrypted_bytes.len() as i64).await?; - // Upload encrypted bytes to Walrus via sidecar (server pays gas) - let sui_key = state.config.sui_private_key.as_deref().ok_or_else(|| { - AppError::Internal("SERVER_SUI_PRIVATE_KEY not configured for Walrus upload".into()) - })?; + // Upload encrypted bytes to Walrus via sidecar (pool key pays gas) + let sui_key = state.key_pool.next() + .map(|s| s.to_string()) + .ok_or_else(|| AppError::Internal("No Sui keys configured (set SERVER_SUI_PRIVATE_KEYS or SERVER_SUI_PRIVATE_KEY)".into()))?; let upload = walrus::upload_blob( &state.http_client, @@ -324,7 +324,7 @@ pub async fn remember_manual( &encrypted_bytes, 50, owner, - sui_key, + &sui_key, namespace, &state.config.package_id, ) From cc7877109b43ae8f1820cbccef4bb30828bdaa2b Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 1 Apr 2026 13:33:40 +0700 Subject: [PATCH 09/10] refactor(openclaw-plugin): extract magic numbers to shared constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate scattered numeric literals into src/constants.ts with documentation for each value's purpose and which modules consume it. No behavior change — pure refactor. --- .../openclaw-memory-memwal/src/capture.ts | 7 +-- .../openclaw-memory-memwal/src/constants.ts | 60 +++++++++++++++++++ packages/openclaw-memory-memwal/src/format.ts | 12 +++- .../src/hooks/recall.ts | 3 +- .../src/tools/search.ts | 3 +- .../openclaw-memory-memwal/src/tools/store.ts | 7 ++- 6 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 packages/openclaw-memory-memwal/src/constants.ts diff --git a/packages/openclaw-memory-memwal/src/capture.ts b/packages/openclaw-memory-memwal/src/capture.ts index 4749c09a..aa4dd4fd 100644 --- a/packages/openclaw-memory-memwal/src/capture.ts +++ b/packages/openclaw-memory-memwal/src/capture.ts @@ -6,12 +6,7 @@ * Based on patterns from LanceDB's shouldCapture() implementation. */ -// ============================================================================ -// Constants -// ============================================================================ - -const MIN_CAPTURE_LENGTH = 30; -const MAX_EMOJI_COUNT = 3; +import { MIN_CAPTURE_LENGTH, MAX_EMOJI_COUNT } from "./constants.js"; /** Filler patterns — exact-match trivial responses. */ const FILLER_PATTERN = /^(ok|okay|sure|thanks|thank you|thx|yes|yep|yeah|no|nope|nah|got it|hmm|hm|ah|oh|lol|haha|nice|cool|great|right|alright|fine|k|kk)\s*[.!?]*$/i; diff --git a/packages/openclaw-memory-memwal/src/constants.ts b/packages/openclaw-memory-memwal/src/constants.ts new file mode 100644 index 00000000..d8c0a3e7 --- /dev/null +++ b/packages/openclaw-memory-memwal/src/constants.ts @@ -0,0 +1,60 @@ +/** + * Shared numeric constants used across the plugin. + * + * Centralised here to avoid magic numbers scattered in business logic. + * Each constant documents its purpose and which modules consume it. + */ + +// ============================================================================ +// Capture filtering (capture.ts) +// ============================================================================ + +/** Minimum character length for a message to be considered capturable. */ +export const MIN_CAPTURE_LENGTH = 30; + +/** Messages with more emoji than this are treated as reactions, not facts. */ +export const MAX_EMOJI_COUNT = 3; + +// ============================================================================ +// Text extraction (format.ts) +// ============================================================================ + +/** Messages shorter than this (after tag stripping) are dropped as trivial. */ +export const MIN_EXTRACTED_TEXT_LENGTH = 10; + +// ============================================================================ +// Recall hook (hooks/recall.ts) +// ============================================================================ + +/** Prompts shorter than this skip the recall round-trip entirely. */ +export const MIN_PROMPT_LENGTH = 10; + +// ============================================================================ +// Store tool (tools/store.ts) +// ============================================================================ + +/** Minimum trimmed length for text submitted to memory_store. */ +export const MIN_STORE_TEXT_LENGTH = 3; + +/** Max extracted facts shown in the store confirmation preview. */ +export const MAX_FACT_PREVIEW_COUNT = 3; + +/** Max characters of raw text shown as fallback preview. */ +export const MAX_TEXT_PREVIEW_LENGTH = 100; + +// ============================================================================ +// Search tool (tools/search.ts) +// ============================================================================ + +/** Default result limit for memory_search when caller omits `limit`. */ +export const DEFAULT_SEARCH_LIMIT = 5; + +// ============================================================================ +// Retry (format.ts → withRetry) +// ============================================================================ + +/** Default number of retry attempts (1 = 2 total tries). */ +export const DEFAULT_RETRY_COUNT = 1; + +/** Default delay in ms between retry attempts. */ +export const DEFAULT_RETRY_DELAY_MS = 2000; diff --git a/packages/openclaw-memory-memwal/src/format.ts b/packages/openclaw-memory-memwal/src/format.ts index e4cb8be3..73bd25ec 100644 --- a/packages/openclaw-memory-memwal/src/format.ts +++ b/packages/openclaw-memory-memwal/src/format.ts @@ -3,6 +3,12 @@ * Shared by hooks, tools, and CLI. */ +import { + MIN_EXTRACTED_TEXT_LENGTH, + DEFAULT_RETRY_COUNT, + DEFAULT_RETRY_DELAY_MS, +} from "./constants.js"; + // ============================================================================ // Constants // ============================================================================ @@ -104,7 +110,7 @@ export function extractMessageTexts( // Strip our injected memory tags to prevent feedback loops, then drop // anything that's empty or trivially short after stripping text = stripMemoryTags(text).trim(); - if (text.length > 10) { + if (text.length > MIN_EXTRACTED_TEXT_LENGTH) { texts.push(text); } } @@ -130,8 +136,8 @@ export function toolError(message: string, err: unknown) { */ export async function withRetry( fn: () => Promise, - retries: number = 1, - delayMs: number = 2000, + retries: number = DEFAULT_RETRY_COUNT, + delayMs: number = DEFAULT_RETRY_DELAY_MS, ): Promise { try { return await fn(); diff --git a/packages/openclaw-memory-memwal/src/hooks/recall.ts b/packages/openclaw-memory-memwal/src/hooks/recall.ts index c6798c55..5b679abb 100644 --- a/packages/openclaw-memory-memwal/src/hooks/recall.ts +++ b/packages/openclaw-memory-memwal/src/hooks/recall.ts @@ -11,8 +11,7 @@ import { resolveAgent } from "../config.js"; import { looksLikeInjection } from "../capture.js"; import { formatMemoriesForPrompt } from "../format.js"; import type { PluginConfig } from "../types.js"; - -const MIN_PROMPT_LENGTH = 10; +import { MIN_PROMPT_LENGTH } from "../constants.js"; /** Register the before_prompt_build hook for auto-recall. */ export function registerRecallHook(api: any, client: MemWal, config: PluginConfig): void { diff --git a/packages/openclaw-memory-memwal/src/tools/search.ts b/packages/openclaw-memory-memwal/src/tools/search.ts index ca987165..4ffc81df 100644 --- a/packages/openclaw-memory-memwal/src/tools/search.ts +++ b/packages/openclaw-memory-memwal/src/tools/search.ts @@ -12,6 +12,7 @@ import { Type } from "@sinclair/typebox"; import { looksLikeInjection } from "../capture.js"; import { escapeForPrompt, toolError } from "../format.js"; import type { PluginConfig } from "../types.js"; +import { DEFAULT_SEARCH_LIMIT } from "../constants.js"; /** Register the memory_search agent tool. */ export function registerSearchTool(api: any, client: MemWal, config: PluginConfig): void { @@ -35,7 +36,7 @@ export function registerSearchTool(api: any, client: MemWal, config: PluginConfi ), }), async execute(_id: string, params: any) { - const { query, limit = 5, namespace } = params; + const { query, limit = DEFAULT_SEARCH_LIMIT, namespace } = params; // LLM may omit namespace (e.g. tools.allow set but hooks disabled) — fall back safely const ns = namespace || config.defaultNamespace; diff --git a/packages/openclaw-memory-memwal/src/tools/store.ts b/packages/openclaw-memory-memwal/src/tools/store.ts index 71e8f861..1fb2c4df 100644 --- a/packages/openclaw-memory-memwal/src/tools/store.ts +++ b/packages/openclaw-memory-memwal/src/tools/store.ts @@ -11,6 +11,7 @@ import { Type } from "@sinclair/typebox"; import { looksLikeInjection } from "../capture.js"; import { toolError } from "../format.js"; import type { PluginConfig } from "../types.js"; +import { MIN_STORE_TEXT_LENGTH, MAX_FACT_PREVIEW_COUNT, MAX_TEXT_PREVIEW_LENGTH } from "../constants.js"; /** Register the memory_store agent tool. */ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig): void { @@ -52,7 +53,7 @@ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig }; } - if (!text || text.trim().length < 3) { + if (!text || text.trim().length < MIN_STORE_TEXT_LENGTH) { return { content: [ { @@ -71,8 +72,8 @@ export function registerStoreTool(api: any, client: MemWal, config: PluginConfig // Show first 3 extracted facts as confirmation, or raw text truncation as fallback const preview = result.facts ?.map((f: any) => f.text) - .slice(0, 3) - .join("; ") ?? text.slice(0, 100); + .slice(0, MAX_FACT_PREVIEW_COUNT) + .join("; ") ?? text.slice(0, MAX_TEXT_PREVIEW_LENGTH); return { content: [ From 5e52008f2324390bff95b06a29e113e56baa64c0 Mon Sep 17 00:00:00 2001 From: ducnmm Date: Wed, 1 Apr 2026 13:51:14 +0700 Subject: [PATCH 10/10] fix(server): restore quota accounting and enoki fallback behavior --- services/server/scripts/sidecar-server.ts | 18 ++++++++++++++++-- services/server/src/main.rs | 2 ++ services/server/src/routes.rs | 18 ++++++++++++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/services/server/scripts/sidecar-server.ts b/services/server/scripts/sidecar-server.ts index ca22dd8b..be3f70ef 100644 --- a/services/server/scripts/sidecar-server.ts +++ b/services/server/scripts/sidecar-server.ts @@ -125,6 +125,10 @@ const enokiNetwork = (process.env.ENOKI_NETWORK || process.env.SUI_NETWORK || "m | "mainnet" | "testnet" | "devnet"; +const ENOKI_FALLBACK_TO_DIRECT_SIGN = (() => { + const raw = (process.env.ENOKI_FALLBACK_TO_DIRECT_SIGN || "true").trim().toLowerCase(); + return raw !== "0" && raw !== "false" && raw !== "no"; +})(); type EnokiDataWrapper = { data: T }; type EnokiSponsorResponse = { bytes: string; digest: string }; @@ -222,8 +226,18 @@ async function executeWithEnokiSponsor(tx: Transaction, signer: Ed25519Keypair, return executed.digest; } catch (err: any) { - console.error(`[enoki-sponsor] sponsor failed: ${err.message}`); - throw err; + const errMsg = err?.message || String(err); + if (!ENOKI_FALLBACK_TO_DIRECT_SIGN) { + console.error(`[enoki-sponsor] sponsor failed and fallback disabled: ${errMsg}`); + throw err; + } + + console.warn(`[enoki-sponsor] sponsor failed, falling back to direct signing: ${errMsg}`); + const direct = await suiClient.signAndExecuteTransaction({ + signer, + transaction: tx, + }); + return direct.digest; } } diff --git a/services/server/src/main.rs b/services/server/src/main.rs index d0b883fd..f64276dd 100644 --- a/services/server/src/main.rs +++ b/services/server/src/main.rs @@ -132,6 +132,8 @@ async fn main() { .route("/api/analyze", post(routes::analyze)) .route("/api/ask", post(routes::ask)) .route("/api/restore", post(routes::restore)) + // Router::layer runs middleware bottom-to-top (last added runs first). + // Keep auth outer so AuthInfo is in request extensions before rate limiting reads it. .layer(middleware::from_fn_with_state( state.clone(), rate_limit::rate_limit_middleware, diff --git a/services/server/src/routes.rs b/services/server/src/routes.rs index 8c30a300..df8396cc 100644 --- a/services/server/src/routes.rs +++ b/services/server/src/routes.rs @@ -891,6 +891,12 @@ pub async fn restore( .flatten() .collect(); + // Preserve encrypted blob sizes so restored rows still contribute to storage quota. + let blob_sizes: std::collections::HashMap = downloaded + .iter() + .map(|(blob_id, data)| (blob_id.clone(), data.len() as i64)) + .collect(); + if downloaded.is_empty() { return Ok(Json(RestoreResponse { restored: 0, @@ -971,8 +977,16 @@ pub async fn restore( let restored = results.len(); for (blob_id, vector) in &results { let id = uuid::Uuid::new_v4().to_string(); - // Restore flow: blob_size not tracked (already counted when first stored) - state.db.insert_vector(&id, owner, namespace, blob_id, vector, 0).await?; + let blob_size = blob_sizes.get(blob_id).copied().unwrap_or_else(|| { + tracing::warn!( + "restore: missing blob size for {}, defaulting to 0 for quota tracking", + blob_id + ); + 0 + }); + state.db + .insert_vector(&id, owner, namespace, blob_id, vector, blob_size) + .await?; } tracing::info!(