diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 7f9faae..83fc6ca 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -41,4 +41,4 @@ ], "deny": [] } -} \ No newline at end of file +} diff --git a/.env.example b/.env.example index 21bf879..d7fd41b 100644 --- a/.env.example +++ b/.env.example @@ -36,4 +36,8 @@ LANGFUSE_HOST=http://localhost:3000 # LOG_LEVEL=info # Environment: development, production (default: production) -# NODE_ENV=production \ No newline at end of file +# NODE_ENV=production + +# Offline fallback file path (default: ~/.claude/telemetry-fallback.jsonl) +# When Langfuse is unreachable, session references are saved here as JSONL +# FALLBACK_FILE=~/.claude/telemetry-fallback.jsonl \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json index 0e62a59..9c8bf49 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -6,10 +6,13 @@ }, "rules": { "comma-dangle": ["error", "always-multiline"], - "space-before-function-paren": ["error", { - "anonymous": "always", - "named": "never", - "asyncArrow": "always" - }] + "space-before-function-paren": [ + "error", + { + "anonymous": "always", + "named": "never", + "asyncArrow": "always" + } + ] } -} \ No newline at end of file +} diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..3c60d3e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,24 @@ +#!/bin/sh +# Pre-commit quality gate — fast local checks only; tests run in CI +set -e + +REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +echo "==> lint" +npm run lint --silent + +echo "==> format:check" +npm run format:check --silent + +echo "==> secret scan (staged files)" +STAGED=$(git diff --cached --name-only --diff-filter=ACM | grep -v '^node_modules/' | grep -v '^\.git/' || true) +if [ -n "$STAGED" ]; then + if echo "$STAGED" | xargs git show HEAD: 2>/dev/null | \ + grep -qE '(api[_-]?key|api[_-]?secret|auth[_-]?token|private[_-]?key|secret[_-]?key)\s*[:=]\s*[A-Za-z0-9+/]{20,}'; then + echo "ERROR: potential secret detected in staged files — commit blocked" + exit 1 + fi +fi + +echo "==> pre-commit passed" diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index c6854a4..c36dfa6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,7 +4,6 @@ about: Create a report to help us improve title: '[BUG] ' labels: 'bug' assignees: '' - --- **Describe the bug** @@ -12,6 +11,7 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Start the telemetry bridge with '...' 2. Configure Claude with '...' 3. Run command '...' @@ -25,20 +25,23 @@ What actually happened instead. **Logs** If applicable, add logs to help explain your problem. + ``` # Paste relevant logs here ``` **Environment (please complete the following information):** - - OS: [e.g. macOS 14.0, Ubuntu 22.04] - - Node.js version: [e.g. 18.17.0] - - Docker version: [e.g. 24.0.7] - - Claude Code version: [run `claude --version`] - - Telemetry Bridge version: [e.g. 3.0.0] - - Langfuse version: [if self-hosted] + +- OS: [e.g. macOS 14.0, Ubuntu 22.04] +- Node.js version: [e.g. 18.17.0] +- Docker version: [e.g. 24.0.7] +- Claude Code version: [run `claude --version`] +- Telemetry Bridge version: [e.g. 3.0.0] +- Langfuse version: [if self-hosted] **Configuration** Relevant environment variables (redact sensitive values): + ```bash OTEL_EXPORTER_OTLP_ENDPOINT= LANGFUSE_HOST= @@ -49,4 +52,4 @@ LANGFUSE_HOST= Add any other context about the problem here. **Possible solution** -If you have an idea how to fix the issue, please describe it. \ No newline at end of file +If you have an idea how to fix the issue, please describe it. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index cea3cf8..493cf34 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,7 +4,6 @@ about: Suggest an idea for this project title: '[FEATURE] ' labels: 'enhancement' assignees: '' - --- **Is your feature request related to a problem? Please describe.** @@ -21,6 +20,7 @@ Explain how this feature would be used and who would benefit from it. **Example** If applicable, provide an example of how this feature would work: + ```javascript // Example code or configuration ``` @@ -29,4 +29,4 @@ If applicable, provide an example of how this feature would work: Add any other context, mockups, or screenshots about the feature request here. **Are you willing to submit a PR?** -Let us know if you'd like to contribute this feature! \ No newline at end of file +Let us know if you'd like to contribute this feature! diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3437fc5..1298bf4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -25,10 +25,11 @@ Please describe the tests that you ran to verify your changes. Provide instructi - [ ] Linting passes (`npm run lint`) **Test Configuration**: -* Node.js version: -* OS: -* Claude Code version: -* Langfuse version (if applicable): + +- Node.js version: +- OS: +- Claude Code version: +- Langfuse version (if applicable): ## Checklist: diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/.quality/guardrails-report.md b/.quality/guardrails-report.md new file mode 100644 index 0000000..c872f8e --- /dev/null +++ b/.quality/guardrails-report.md @@ -0,0 +1,49 @@ +--- +schema_version: 1 +task_id: 'guardrails-baseline' +requires_split_authorship: false +na_controls: + - control: 'parseable_handoff_artifact' + reason: 'Repo does not require split test/implementation authorship.' + - control: 'typescript_type_checking' + reason: 'Repo is JavaScript-only; no TypeScript layer.' +blockers: [] +--- + +# Guardrails Report — claude-code-telemetry + +Generated: 2026-04-03 + +## Policy vs Enforcement Matrix + +| Control | Policy source | Enforcement | Status | Notes | +| ---------------------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Claude Code doc naming (AGENTS.md) | skill:claudecode-conventions | `AGENTS.md` created; `CLAUDE.md` replaced with `@AGENTS.md` | **enforced** | Was blocker before this PR | +| Pinned toolchain manifest | node-ts reference | `.nvmrc` (Node 22); `engines` in `package.json`; lockfile checked in | **partial** | `engines` uses range `>=18`, not exact pin. `.nvmrc` pins 22. No `packageManager` field yet. | +| Local hook wiring | node-ts reference | `.githooks/pre-commit`; `scripts/install-hooks.sh`; `npm run hooks:install` | **enforced** | Developer must run `npm run hooks:install` once after clone | +| Local quality gate | node-ts reference | `scripts/ci/run_quality_gates.sh`; `npm run quality` | **enforced** | Runs format:check + lint + unit tests offline | +| Local security review entrypoint | node-ts reference | `scripts/ci/run_security_review.sh`; `npm run security` | **partial** | `npm audit` requires live network; docker step skips gracefully if no docker; secret scan is local | +| Dependency vulnerability audit | `.github/workflows/security.yml` | CI: `npm audit --audit-level=high`; local: `npm run security` | **partial** | Network-dependent; no local advisory mirror | +| Secret scanning | `.github/workflows/security.yml` | CI: git grep regex; local: pre-commit hook + `npm run security` | **enforced** locally, **partial** in CI | CI scan excludes test/ and \*.json; local hook scans staged files | +| Docker / runtime checks | `Dockerfile`; `.github/workflows/ci.yml` | Non-root user, healthcheck, prod-only deps in Dockerfile; Trivy in CI; local docker build in security script | **partial** | Trivy is CI-only + network; local build smoke check added | +| GitNexus / code intelligence | skill:gitnexus reference | Not installed | **policy-only** | Bootstrap: `npx gitnexus analyze` from repo root. Required before editing shared modules (`sessionHandler.js`, `requestHandlers.js`). | +| TypeScript type checking | N/A | N/A | **N/A** | JavaScript-only repo | +| Parseable handoff artifact | N/A | N/A | **N/A** | No split authorship requirement | + +## Manual Review Obligations + +The following require human review and cannot be automated: + +| Area | Location | Why manual | +| ---------------------------- | ------------------------ | ----------------------------------------------------------------------------- | +| API key auth middleware | `src/server.js` | Must verify request validation is correct for the threat model | +| Outbound HTTP to Langfuse | `src/sessionHandler.js` | Verify no sensitive data leaks beyond what is intentional | +| `FALLBACK_FILE` path | `src/offlineFallback.js` | Env var controls fs write path — verify no path traversal risk | +| `OTLP_RECEIVER_HOST` binding | `src/serverHelpers.js` | Defaults to 127.0.0.1 in dev, 0.0.0.0 in Docker — review for network exposure | + +## Remaining Gaps (not blockers) + +1. **GitNexus not installed** — run `npx gitnexus analyze` to bootstrap code-intelligence index. Required before making changes to widely-used modules. +2. **`engines` range is not exact** — `>=18.0.0` allows drift; `.nvmrc` mitigates for local dev but CI tests across 18/20/22. +3. **`npm audit` is network-dependent** — no local advisory mirror. Acceptable for this project size; mark as `partial`. +4. **`packageManager` field absent** — consider adding `"packageManager": "npm@10.x.x"` to lock the package manager version. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dd89399 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,305 @@ +# Claude Code Telemetry - Setup Guide for Non-Technical Users 🤖 + +Hi! I'm Claude. This folder contains a tool that tracks how much money your team spends on AI. I'll help you set it up step by step. + +## 🎯 What You'll Get + +A dashboard that shows: + +- **Daily AI costs** - "We spent $47.32 on AI assistance today" +- **Who uses AI most** - "Sarah used AI 23 times today for report writing" +- **What tasks cost most** - "Code reviews cost $5-10, emails cost $0.50" +- **Usage trends** - "AI usage is up 40% this week" + +## ⚠️ Before We Start - You Need Docker + +**Docker is required!** Think of Docker as a special program that runs other programs in isolated boxes. Here's how to get it: + +### How to Check if You Have Docker: + +1. Look for a **whale icon** 🐳 in your menu bar (top of screen on Mac, bottom right on Windows) +2. If you see the whale, you're ready! +3. If not, you need to install Docker first + +### How to Install Docker: + +1. Go to: https://www.docker.com/products/docker-desktop +2. Click the big blue **"Download Docker Desktop"** button +3. Choose your system: + - **Mac**: Download for Mac (Intel chip or Apple Silicon) + - **Windows**: Download for Windows +4. Install it like any other program (double-click the downloaded file) +5. Open Docker Desktop after installation +6. Wait for the whale icon to appear in your menu bar +7. The whale should be steady (not animated) when Docker is ready + +### Docker Not Starting? + +- **Mac**: Check Applications folder and double-click Docker +- **Windows**: Check Start Menu and click Docker Desktop +- **Still stuck?** Restart your computer and try again + +## 🚀 Getting Started - Just Ask Me! + +**Once Docker is running** (whale icon visible), type this exactly: + +``` +Set up the telemetry dashboard +``` + +## 📝 What I'll Do - Complete Setup Process + +When you ask me to set up, I'll run the quickstart script that does everything automatically: + +```bash +./quickstart.sh +``` + +This single command will: + +- Check if Docker is installed +- Download everything needed +- Start all services +- Create your login credentials +- Configure Claude to send telemetry + +Here's EXACTLY what you'll see: + +### Step 1: I'll Check Docker + +``` +Me: "First, I need to check if Docker is installed and running..." +[I run: docker --version] + +IF DOCKER IS MISSING: +Me: "❌ Docker isn't installed yet. Please follow the Docker installation + instructions at the top of this guide (see 'Before We Start' section). + + Quick reminder: + 1. Go to https://docker.com/products/docker-desktop + 2. Download and install Docker Desktop + 3. Look for the whale icon 🐳 in your menu bar + 4. Once you see the whale, tell me 'Docker is ready' and we'll continue!" + +IF DOCKER IS INSTALLED BUT NOT RUNNING: +Me: "❌ Docker is installed but not running. Please: + 1. Find and open Docker Desktop + 2. Wait for the whale icon 🐳 to appear in your menu bar + 3. Make sure the whale is steady (not animating) + 4. Tell me 'Docker is running' when ready!" + +IF DOCKER IS READY: +Me: "✅ Great! Docker is installed and running. Let's continue..." +``` + +### Step 2: I'll Download Everything + +``` +Me: "Now I'll download all the files we need. This takes about 2 minutes..." +[Progress messages appear] +Me: "✅ Downloaded 1 of 5 components..." +Me: "✅ Downloaded 2 of 5 components..." +[etc.] +``` + +### Step 3: I'll Start the Dashboard + +``` +Me: "Starting your dashboard services..." +Me: "✅ Database started" +Me: "✅ Analytics engine started" +Me: "✅ Web dashboard started" +Me: "✅ Telemetry collector started" +Me: "Everything is running!" +``` + +### Step 4: I'll Set Up Your Login + +``` +Me: "Creating your login credentials..." +Me: " +YOUR DASHBOARD ACCESS: +📍 Website: http://localhost:3000 +📧 Email: admin@example.com +🔑 Password: [unique password generated for you] + +I've also saved these in a file called 'langfuse-credentials.txt' +" +``` + +### Step 5: I'll Configure Claude + +``` +Me: "Now I'll set up Claude to send telemetry data..." +Me: "Running: source claude-telemetry.env" +Me: "✅ Claude is now configured to track usage!" +``` + +### Step 6: I'll Help You Test It + +``` +Me: "Let's make sure you can access your dashboard: + 1. Click this link: http://localhost:3000 + 2. You should see a login page + 3. Copy and paste the email and password I gave you + 4. Click 'Sign In' + + Do you see the dashboard? (yes/no)" +``` + +## 🚨 When Things Go Wrong - I'll Fix Them! + +### "I see an error message" + +Tell me: "I got an error" and copy/paste what you see. I'll: + +1. Read the error +2. Explain what it means in plain English +3. Fix it for you +4. Try again + +### "Nothing happens when I click the link" + +Tell me: "The website won't open". I'll: + +1. Check if services are running +2. Restart them if needed +3. Give you a new link to try +4. Test it with you + +### "I can't find the whale icon" + +Tell me: "I don't see Docker". I'll: + +1. Help you check if it's installed +2. Guide you through installation with pictures +3. Show you exactly where to look +4. Wait while you install it + +### "The password doesn't work" + +Tell me: "I can't log in". I'll: + +1. Show you the password again +2. Make sure you're copying it exactly +3. Create a new password if needed +4. Walk you through typing it + +## 📊 Using Your Dashboard - What You'll See + +Once logged in, you'll see: + +### Main Dashboard Page + +``` +Today's AI Costs: $47.32 +Total This Week: $156.89 +Most Active User: sarah@company.com (45 requests) +Most Expensive Task: "Analyze sales data" ($12.47) +``` + +### Traces Page (Individual Conversations) + +``` +10:32 AM - john@company.com - "Write email to client" - $0.73 +10:45 AM - sarah@company.com - "Debug Python code" - $3.21 +11:02 AM - mike@company.com - "Summarize meeting notes" - $1.15 +``` + +### Sessions Page (Work Blocks) + +``` +Morning Session (9 AM - 10 AM) - Total: $15.43 +- 5 code reviews +- 3 emails written +- 2 reports analyzed +``` + +## 🛑 Stopping Everything + +When you're done, tell me: + +``` +Stop the dashboard +``` + +I'll: + +1. Save any pending data +2. Shut down all services +3. Confirm everything stopped +4. Tell you how to start again later + +## 🔄 Starting Again Later + +Next time, just tell me: + +``` +Start my dashboard +``` + +I'll: + +1. Check everything is ready +2. Start all services +3. Give you the login link +4. Make sure you can access it + +## ❓ Questions I Can Answer + +- "How much did we spend on AI yesterday?" +- "Show me who uses AI the most" +- "What tasks are most expensive?" +- "How do I download a cost report?" +- "Can I add other team members?" +- "How do I see what questions were asked?" + +## 🎉 Success Checklist + +You'll know everything worked when: + +- ✅ You see the whale icon in your menu bar +- ✅ You can open http://localhost:3000 +- ✅ You can log in with the email/password +- ✅ You see cost data appearing +- ✅ The numbers update when you use Claude + +## 💡 Important Notes + +- **Docker is required** - This won't work without Docker Desktop installed +- **Your data stays on your computer** - Nothing goes to the cloud +- **The dashboard runs locally** - Only you can see it +- **Costs appear after you use Claude** - Not from past usage +- **Keep Docker running** - The whale icon must stay in your menu bar + +## 🐳 Quick Docker Check + +Before starting, make sure: + +1. Docker Desktop is installed (see instructions at top) +2. The whale icon 🐳 is visible in your menu bar +3. The whale is steady (not animating) + +If any of these aren't true, go back to the "Before We Start" section. + +--- + +**Ready? Once Docker is running, just type: `Set up the telemetry dashboard`** + +I'll handle everything else and guide you through each step! + +--- + +## Developer Notes + +> This section is for contributors. End users can ignore everything below. + +- Source: `src/` — OTLP receiver + Langfuse bridge + session handler +- Tests: `npm test` (unit), `npm run test:integration` (needs Langfuse) +- Lint + format: `npm run lint`, `npm run format:check` +- Local quality gate: `npm run quality` — runs lint, format check, unit tests +- Local security review: `npm run security` — runs npm audit + secret scan +- Install git hooks: `bash scripts/install-hooks.sh` +- See `CONTRIBUTING.md` and `docs/DEVELOPMENT.md` for full contributor guide +- Offline fallback: when Langfuse is unreachable, session references are saved to `~/.claude/telemetry-fallback.jsonl` +- GitNexus index: run `npx gitnexus analyze` from repo root for code-intelligence before editing shared modules diff --git a/CHANGELOG.md b/CHANGELOG.md index e40980f..59cad2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,4 +59,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.0.0] - 2025-08-03 --- + Generated by [git-cliff](https://github.com/orhun/git-cliff) diff --git a/CLAUDE.md b/CLAUDE.md index bdbc404..43c994c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,260 +1 @@ -# Claude Code Telemetry - Setup Guide for Non-Technical Users 🤖 - -Hi! I'm Claude. This folder contains a tool that tracks how much money your team spends on AI. I'll help you set it up step by step. - -## 🎯 What You'll Get - -A dashboard that shows: -- **Daily AI costs** - "We spent $47.32 on AI assistance today" -- **Who uses AI most** - "Sarah used AI 23 times today for report writing" -- **What tasks cost most** - "Code reviews cost $5-10, emails cost $0.50" -- **Usage trends** - "AI usage is up 40% this week" - -## ⚠️ Before We Start - You Need Docker - -**Docker is required!** Think of Docker as a special program that runs other programs in isolated boxes. Here's how to get it: - -### How to Check if You Have Docker: -1. Look for a **whale icon** 🐳 in your menu bar (top of screen on Mac, bottom right on Windows) -2. If you see the whale, you're ready! -3. If not, you need to install Docker first - -### How to Install Docker: -1. Go to: https://www.docker.com/products/docker-desktop -2. Click the big blue **"Download Docker Desktop"** button -3. Choose your system: - - **Mac**: Download for Mac (Intel chip or Apple Silicon) - - **Windows**: Download for Windows -4. Install it like any other program (double-click the downloaded file) -5. Open Docker Desktop after installation -6. Wait for the whale icon to appear in your menu bar -7. The whale should be steady (not animated) when Docker is ready - -### Docker Not Starting? -- **Mac**: Check Applications folder and double-click Docker -- **Windows**: Check Start Menu and click Docker Desktop -- **Still stuck?** Restart your computer and try again - -## 🚀 Getting Started - Just Ask Me! - -**Once Docker is running** (whale icon visible), type this exactly: -``` -Set up the telemetry dashboard -``` - -## 📝 What I'll Do - Complete Setup Process - -When you ask me to set up, I'll run the quickstart script that does everything automatically: - -```bash -./quickstart.sh -``` - -This single command will: -- Check if Docker is installed -- Download everything needed -- Start all services -- Create your login credentials -- Configure Claude to send telemetry - -Here's EXACTLY what you'll see: - -### Step 1: I'll Check Docker -``` -Me: "First, I need to check if Docker is installed and running..." -[I run: docker --version] - -IF DOCKER IS MISSING: -Me: "❌ Docker isn't installed yet. Please follow the Docker installation - instructions at the top of this guide (see 'Before We Start' section). - - Quick reminder: - 1. Go to https://docker.com/products/docker-desktop - 2. Download and install Docker Desktop - 3. Look for the whale icon 🐳 in your menu bar - 4. Once you see the whale, tell me 'Docker is ready' and we'll continue!" - -IF DOCKER IS INSTALLED BUT NOT RUNNING: -Me: "❌ Docker is installed but not running. Please: - 1. Find and open Docker Desktop - 2. Wait for the whale icon 🐳 to appear in your menu bar - 3. Make sure the whale is steady (not animating) - 4. Tell me 'Docker is running' when ready!" - -IF DOCKER IS READY: -Me: "✅ Great! Docker is installed and running. Let's continue..." -``` - -### Step 2: I'll Download Everything -``` -Me: "Now I'll download all the files we need. This takes about 2 minutes..." -[Progress messages appear] -Me: "✅ Downloaded 1 of 5 components..." -Me: "✅ Downloaded 2 of 5 components..." -[etc.] -``` - -### Step 3: I'll Start the Dashboard -``` -Me: "Starting your dashboard services..." -Me: "✅ Database started" -Me: "✅ Analytics engine started" -Me: "✅ Web dashboard started" -Me: "✅ Telemetry collector started" -Me: "Everything is running!" -``` - -### Step 4: I'll Set Up Your Login -``` -Me: "Creating your login credentials..." -Me: " -YOUR DASHBOARD ACCESS: -📍 Website: http://localhost:3000 -📧 Email: admin@example.com -🔑 Password: [unique password generated for you] - -I've also saved these in a file called 'langfuse-credentials.txt' -" -``` - -### Step 5: I'll Configure Claude -``` -Me: "Now I'll set up Claude to send telemetry data..." -Me: "Running: source claude-telemetry.env" -Me: "✅ Claude is now configured to track usage!" -``` - -### Step 6: I'll Help You Test It -``` -Me: "Let's make sure you can access your dashboard: - 1. Click this link: http://localhost:3000 - 2. You should see a login page - 3. Copy and paste the email and password I gave you - 4. Click 'Sign In' - - Do you see the dashboard? (yes/no)" -``` - -## 🚨 When Things Go Wrong - I'll Fix Them! - -### "I see an error message" -Tell me: "I got an error" and copy/paste what you see. I'll: -1. Read the error -2. Explain what it means in plain English -3. Fix it for you -4. Try again - -### "Nothing happens when I click the link" -Tell me: "The website won't open". I'll: -1. Check if services are running -2. Restart them if needed -3. Give you a new link to try -4. Test it with you - -### "I can't find the whale icon" -Tell me: "I don't see Docker". I'll: -1. Help you check if it's installed -2. Guide you through installation with pictures -3. Show you exactly where to look -4. Wait while you install it - -### "The password doesn't work" -Tell me: "I can't log in". I'll: -1. Show you the password again -2. Make sure you're copying it exactly -3. Create a new password if needed -4. Walk you through typing it - -## 📊 Using Your Dashboard - What You'll See - -Once logged in, you'll see: - -### Main Dashboard Page -``` -Today's AI Costs: $47.32 -Total This Week: $156.89 -Most Active User: sarah@company.com (45 requests) -Most Expensive Task: "Analyze sales data" ($12.47) -``` - -### Traces Page (Individual Conversations) -``` -10:32 AM - john@company.com - "Write email to client" - $0.73 -10:45 AM - sarah@company.com - "Debug Python code" - $3.21 -11:02 AM - mike@company.com - "Summarize meeting notes" - $1.15 -``` - -### Sessions Page (Work Blocks) -``` -Morning Session (9 AM - 10 AM) - Total: $15.43 -- 5 code reviews -- 3 emails written -- 2 reports analyzed -``` - -## 🛑 Stopping Everything - -When you're done, tell me: -``` -Stop the dashboard -``` - -I'll: -1. Save any pending data -2. Shut down all services -3. Confirm everything stopped -4. Tell you how to start again later - -## 🔄 Starting Again Later - -Next time, just tell me: -``` -Start my dashboard -``` - -I'll: -1. Check everything is ready -2. Start all services -3. Give you the login link -4. Make sure you can access it - -## ❓ Questions I Can Answer - -- "How much did we spend on AI yesterday?" -- "Show me who uses AI the most" -- "What tasks are most expensive?" -- "How do I download a cost report?" -- "Can I add other team members?" -- "How do I see what questions were asked?" - -## 🎉 Success Checklist - -You'll know everything worked when: -- ✅ You see the whale icon in your menu bar -- ✅ You can open http://localhost:3000 -- ✅ You can log in with the email/password -- ✅ You see cost data appearing -- ✅ The numbers update when you use Claude - -## 💡 Important Notes - -- **Docker is required** - This won't work without Docker Desktop installed -- **Your data stays on your computer** - Nothing goes to the cloud -- **The dashboard runs locally** - Only you can see it -- **Costs appear after you use Claude** - Not from past usage -- **Keep Docker running** - The whale icon must stay in your menu bar - -## 🐳 Quick Docker Check - -Before starting, make sure: -1. Docker Desktop is installed (see instructions at top) -2. The whale icon 🐳 is visible in your menu bar -3. The whale is steady (not animating) - -If any of these aren't true, go back to the "Before We Start" section. - ---- - -**Ready? Once Docker is running, just type: `Set up the telemetry dashboard`** - -I'll handle everything else and guide you through each step! \ No newline at end of file +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2eca539..07aa126 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,7 @@ Thank you for your interest in contributing to Claude Code Telemetry! This docum This project maintains two distinct documentation files for different audiences: ### CLAUDE.md (User Documentation) + - **Purpose**: Guide non-technical users through setup and usage - **Audience**: End users who want to track their Claude Code usage - **Content**: Step-by-step instructions, troubleshooting, visual guides @@ -14,6 +15,7 @@ This project maintains two distinct documentation files for different audiences: - **Note**: This file is automatically read by Claude when users ask for help ### DEVELOPMENT.md (Developer Documentation) + - **Purpose**: Technical guide for contributors and developers - **Audience**: Engineers working on or with the codebase - **Content**: Architecture, APIs, implementation details, debugging info @@ -87,4 +89,3 @@ npm run format ## Code of Conduct Please be respectful and professional in all interactions. We aim to maintain a welcoming and inclusive community. - diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b5d92eb..fdb8ae5 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -7,17 +7,20 @@ This guide helps you quickly understand and resume work on the telemetry bridge. **Purpose**: Bridge that captures Claude Code's telemetry and forwards to Langfuse for LLM observability. **Architecture**: + ``` Claude Code → OTLP/HTTP → Bridge Server → Langfuse API (JSON logs) (Parse & Map) (Traces/Spans) ``` **Quick Setup**: Run `./quickstart.sh` for a complete setup with: + - Langfuse stack (PostgreSQL, ClickHouse, Redis, MinIO) - Unique credentials generated automatically - Telemetry bridge configured and ready **Key Modules**: + - `src/server.js` - Main OTLP server (port 4318) - `src/sessionHandler.js` - Session lifecycle management - `src/eventProcessor.js` - Maps Claude events to Langfuse @@ -37,6 +40,7 @@ Claude Code → OTLP/HTTP → Bridge Server → Langfuse API 4. **Custom event names** - Uses `claude_code.*` namespace ### Required Environment Variables (All 6) + ```bash export CLAUDE_CODE_ENABLE_TELEMETRY=1 export OTEL_LOGS_EXPORTER=otlp @@ -47,9 +51,10 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 # NO DEFAULT! ``` ### Standard Attributes on All Events/Metrics + - `session.id` - Unique session identifier - `organization.id` - Organization UUID -- `user.account_uuid` - User account UUID +- `user.account_uuid` - User account UUID - `user.email` - User email address - `terminal.type` - Terminal type (e.g., "vscode") - `app.version` - Claude Code version @@ -57,6 +62,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 # NO DEFAULT! ## 📊 Data Flow ### Events (via Logs) + 1. `claude_code.user_prompt` - User input received - Contains: prompt, prompt_length, event.timestamp 2. `claude_code.api_request` - Model calls (Haiku + Opus) @@ -69,6 +75,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 # NO DEFAULT! - Contains: decision, source, tool_name ### Metrics + - `claude_code.cost.usage` - USD per model - `claude_code.token.usage` - Token breakdown (input/output/cacheRead/cacheCreation) - `claude_code.lines_of_code.count` - Code changes (added/removed) @@ -79,12 +86,14 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 # NO DEFAULT! - `claude_code.session.count` - Session starts ### Session Lifecycle + - Auto-created on first event - 1-hour timeout for cleanup (configurable via SESSION_TIMEOUT) - Tracks: total cost, tokens, cache usage, tool usage, code changes - Creates session summary with quality and efficiency scores ### Langfuse Mapping + - **Traces**: One per conversation + session summary - **Generations**: For each API call with full token/cost data - **Events**: For tools, decisions, errors, and milestones @@ -93,6 +102,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 # NO DEFAULT! ## 🧪 Testing ### Test Structure + ``` test/ ├── unit/ # Mocked tests - fast, isolated @@ -101,6 +111,7 @@ test/ ``` ### Running Tests + ```bash # Unit tests only (no external dependencies) npm run test:unit @@ -115,6 +126,7 @@ npm test ``` ### Manual Testing + ```bash # Terminal 1: Start server npm start @@ -131,6 +143,7 @@ claude "What is 2+2?" ``` ### Integration Test Features + - **LangfuseTestClient**: Direct API access for verification - **OTLP Test Data Builders**: Consistent test fixtures - **E2E Tests**: Run real Claude commands @@ -139,24 +152,28 @@ claude "What is 2+2?" ## 🐛 Debugging ### No Telemetry Received + 1. Verify ALL 6 env vars set (use server startup banner) 2. Check endpoint has no typos (common: missing http://) 3. Confirm server health: `curl http://localhost:4318/health` 4. Enable debug: `LOG_LEVEL=debug npm start` ### Missing Data in Langfuse + - Run validation: `node test/helpers/validate-langfuse.js` - Check for conversation traces vs session-summary traces - Verify all standard attributes are present - Ensure metrics are being sent (not just logs) ### Common Issues + - **No generations**: Check logs are being sent, not just metrics - **No scores**: Created only on session finalization - **Missing cache tokens**: Verify token metrics include all types - **No events**: Tool usage creates events, simple prompts don't ### Langfuse Connection Issues + - Check `.env` has valid keys - Verify `LANGFUSE_HOST` is correct - Use `scripts/debug-generations.js` for manual testing @@ -164,6 +181,7 @@ claude "What is 2+2?" ## 🔧 Development Workflow ### Making Changes + 1. **Event Processing**: Edit `src/eventProcessor.js` 2. **Metrics**: Edit `src/metricsProcessor.js` 3. **Session Logic**: Edit `src/sessionHandler.js` @@ -171,6 +189,7 @@ claude "What is 2+2?" 5. **New Endpoints**: Edit `src/server.js` ### Adding New Event Types + ```javascript // In eventProcessor.js case 'claude_code.new_event': @@ -183,6 +202,7 @@ case 'claude_code.new_event': ``` ### Testing Changes + 1. Write unit test with mocks 2. Write integration test with real Langfuse 3. Test manually with Claude (see Quick Commands) @@ -191,6 +211,7 @@ case 'claude_code.new_event': ## 📝 Recent Changes & Context ### Major Refactoring (Latest) + 1. **Comprehensive telemetry capture** - All Claude Code events and metrics 2. **Fixed Langfuse SDK usage** - Use traceId, not parentObservationId 3. **True integration tests** - Test with real Langfuse API @@ -199,17 +220,20 @@ case 'claude_code.new_event': 6. **Cache token support** - Track cache read/creation separately ### Architecture Decision + - **Stateful session management** is the correct approach - Session aggregation provides the actual value customers need - Complexity is worth it for actionable insights (costs, efficiency, productivity) - Focus on making the aggregated data more valuable, not simpler ### Test Coverage + - **96%+ coverage** on business logic - **Real integration tests** catch actual issues - **E2E tests** validate full flow ### Known Limitations + - Sessions accumulate in memory (1-hour cleanup helps manage this) - Some metrics rarely observed (PR count, active time) - Session summary created on timeout or graceful shutdown @@ -217,6 +241,7 @@ case 'claude_code.new_event': ## 🎯 Future Improvements That Actually Matter ### Customer-Focused Features + 1. **Cost Alerts**: Notify when session exceeds threshold 2. **Daily Reports**: Email summary of team AI usage 3. **Efficiency Tips**: "You could save 80% by using cache" @@ -224,6 +249,7 @@ case 'claude_code.new_event': 5. **ROI Calculator**: Hours saved vs dollars spent ### What NOT to Build + - Alternative architectures (stateless, event sourcing, etc.) - Complex configuration options - Multiple storage backends @@ -273,7 +299,8 @@ lsof -i:4318 --- -**Remember**: +**Remember**: + 1. Claude's OTLP implementation is non-standard - no defaults! 2. Always test with real Claude binary when making changes 3. Integration tests are your friend - they catch real issues diff --git a/README.md b/README.md index 585df33..0552aa8 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ https://github.com/user-attachments/assets/2634cec3-94af-4a2d-90da-44cd641f1746 - ## 🎯 What This Actually Does Claude Code Telemetry is a lightweight bridge that captures telemetry data from Claude Code and forwards it to Langfuse for visualization. You get: @@ -31,14 +30,17 @@ Claude Code Telemetry is a lightweight bridge that captures telemetry data from The original motivation from the author was that when using Claude Code Pro/Max, it didn't have good options for telemetry out of the box compared to API-based requests that can be integrated with various solutions and wanted to provide a secure turnkey local setup for people using Claude Code to benefit from. ### 🏗️ Built on Standards + Uses **OpenTelemetry** for data collection, **Langfuse** for visualization, and **Claude's native observability** APIs. No proprietary formats, no vendor lock-in. ## 🚀 Quick Start (30 seconds) ### Prerequisites + 🐳 **Docker Desktop** - [Install here](https://docker.com/products/docker-desktop) if you don't see the whale icon in your menu bar ### Setup + ```bash # Clone and enter directory git clone https://github.com/lainra/claude-code-telemetry && cd claude-code-telemetry @@ -56,7 +58,9 @@ claude "What is 2+2?" **That's it!** View your dashboard at http://localhost:3000 ### Need Help? + Let Claude guide you through the setup: + ```bash claude "Set up the telemetry dashboard" ``` @@ -64,7 +68,9 @@ claude "Set up the telemetry dashboard" ## 📸 What You'll See in Langfuse ### Session View + Every conversation becomes a trackable session: + ``` Session: 4:32 PM - 5:15 PM (43 minutes) ├── Total Cost: $18.43 @@ -79,7 +85,9 @@ Session: 4:32 PM - 5:15 PM (43 minutes) ``` ### Individual API Calls + Full details for every Claude interaction: + ``` 4:45 PM - claude-3-opus-20240229 ├── Input: 12,453 tokens (8,234 from cache) @@ -90,7 +98,9 @@ Full details for every Claude interaction: ``` ### Cost Breakdown + Track spending by model and user: + ``` Today's Usage: ├── Total: $67.43 @@ -113,6 +123,7 @@ Claude Code → OpenTelemetry → Telemetry Bridge → Langfuse ``` The bridge: + 1. Listens for OpenTelemetry data from Claude Code 2. Enriches it with session context 3. Forwards to Langfuse for visualization @@ -121,6 +132,7 @@ The bridge: ## 🌟 What This Tool Is (and Isn't) ### ✅ What It Does: + - **Tracks costs** - Know exactly what you're spending - **Shows usage patterns** - See when and how Claude is used - **Groups work sessions** - Understand complete tasks, not just individual calls @@ -128,6 +140,7 @@ The bridge: - **Runs locally** - Your data stays on your infrastructure ### ❌ What It Doesn't Do: + - **Measure productivity** - Can't tell if you're working faster - **Analyze code quality** - Doesn't evaluate AI-generated code - **Provide strategic insights** - Just shows raw data, not recommendations @@ -137,13 +150,17 @@ The bridge: ## 🛠️ Installation Options ### Option 1: Full Stack (Recommended) + Includes Langfuse dashboard + telemetry bridge: + ```bash ./quickstart.sh ``` ### Option 2: Bridge Only (Manual w/NPM) + Already have Langfuse? Just run the bridge: + ```bash # Configure your existing Langfuse credentials export LANGFUSE_PUBLIC_KEY=your-public-key @@ -156,7 +173,9 @@ npm start ``` ### Option 3: Bridge Only (Docker) + Already have Langfuse? Run the bridge in Docker: + ```bash # Create .env file with your Langfuse credentials cp .env.example .env @@ -174,12 +193,12 @@ docker compose up telemetry-bridge ## 🎛️ Configuration -| Setting | Default | Description | -|---------|---------|-------------| -| `SESSION_TIMEOUT` | 1 hour | Groups related work into sessions | -| `OTLP_RECEIVER_PORT` | 4318 | OpenTelemetry standard port | -| `LANGFUSE_HOST` | http://localhost:3000 | Langfuse dashboard URL | -| `LOG_LEVEL` | info | Logging verbosity | +| Setting | Default | Description | +| -------------------- | --------------------- | --------------------------------- | +| `SESSION_TIMEOUT` | 1 hour | Groups related work into sessions | +| `OTLP_RECEIVER_PORT` | 4318 | OpenTelemetry standard port | +| `LANGFUSE_HOST` | http://localhost:3000 | Langfuse dashboard URL | +| `LOG_LEVEL` | info | Logging verbosity | See `.env.example` for all options. @@ -198,8 +217,9 @@ See `.env.example` for all options. ## 🤔 Should You Use This? **Use this if you want to:** + - Track Claude Code costs across your team -- Understand usage patterns and peak times +- Understand usage patterns and peak times - Have transparency into AI tool spending - Keep telemetry data on your own infrastructure diff --git a/SECURITY.md b/SECURITY.md index e5c72c3..9ae1bb4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -35,6 +35,7 @@ We take security vulnerabilities seriously. Please report security issues privat ### Security Measures This project implements: + - No storage of sensitive data (prompts optional) - Local-only operation by default - Secure credential handling via environment variables @@ -47,4 +48,4 @@ This project implements: - Public disclosure will occur after patches are available - Credit will be given to reporters (unless anonymity requested) -Thank you for helping keep Claude Code Telemetry secure! \ No newline at end of file +Thank you for helping keep Claude Code Telemetry secure! diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index e9528c5..e22dcd1 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -5,6 +5,7 @@ This document explains all environment variables needed to configure Claude Code ## Overview Claude Code uses OpenTelemetry (OTLP) to send telemetry data. The telemetry includes: + - **Logs**: User prompts, API requests, tool usage, and errors - **Metrics**: Cost tracking, token usage, code modifications, and performance data @@ -13,36 +14,42 @@ Claude Code uses OpenTelemetry (OTLP) to send telemetry data. The telemetry incl These variables MUST be set for telemetry to work: ### `CLAUDE_CODE_ENABLE_TELEMETRY=1` + - **Purpose**: Master switch that enables all telemetry - **Required**: YES - **Default**: Not set (telemetry disabled) - **Note**: Without this, no telemetry will be sent regardless of other settings ### `OTEL_LOGS_EXPORTER=otlp` + - **Purpose**: Enables OTLP exporter for logs (events) - **Required**: YES - **Default**: Not set - **Note**: Needed to capture user prompts, API calls, and tool usage ### `OTEL_METRICS_EXPORTER=otlp` + - **Purpose**: Enables OTLP exporter for metrics - **Required**: YES - **Default**: Not set - **Note**: Needed to capture cost, token usage, and performance metrics ### `OTEL_EXPORTER_OTLP_PROTOCOL=http/json` + - **Purpose**: Sets the OTLP protocol format - **Required**: YES - **Default**: May default to protobuf (which won't work) - **Note**: MUST be `http/json` - Claude doesn't support protobuf ### `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/json` + - **Purpose**: Sets the OTLP protocol specifically for metrics - **Required**: YES - **Default**: May default to protobuf - **Note**: MUST be `http/json` for metrics to work ### `OTEL_EXPORTER_OTLP_ENDPOINT` + - **Purpose**: Base endpoint for all OTLP data - **Required**: YES (Claude Code does not provide a default) - **Default**: None - must be explicitly set @@ -54,6 +61,7 @@ These variables MUST be set for telemetry to work: These variables have sensible defaults: ### `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` + - **Purpose**: Specific endpoint for logs - **Required**: NO - **Default**: `${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs` @@ -61,6 +69,7 @@ These variables have sensible defaults: - **Note**: Only set if you need a different endpoint than the base ### `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` + - **Purpose**: Specific endpoint for metrics - **Required**: NO - **Default**: `${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics` @@ -68,6 +77,7 @@ These variables have sensible defaults: - **Note**: Only set if you need a different endpoint than the base ### `OTEL_LOG_USER_PROMPTS` + - **Purpose**: Include full user prompt text in telemetry - **Required**: NO - **Default**: `1` (enabled in quickstart setup) @@ -75,9 +85,17 @@ These variables have sensible defaults: - **Privacy Note**: When enabled, full prompt text is sent to the telemetry server - **Note**: The quickstart script enables this by default for better observability +### `FALLBACK_FILE` + +- **Purpose**: Path to the offline fallback JSONL file where session references are saved when Langfuse is unreachable +- **Required**: NO +- **Default**: `~/.claude/telemetry-fallback.jsonl` +- **Note**: When Langfuse flush fails after retries, a lightweight session reference is appended to this file. The reference includes sessionId, userId, and basic stats so the full transcript can be recovered from local Claude session data. + ## Quick Start Examples ### Minimal Configuration (Recommended) + ```bash # Copy and paste this block: export CLAUDE_CODE_ENABLE_TELEMETRY=1 @@ -90,6 +108,7 @@ export OTEL_LOG_USER_PROMPTS=1 ``` ### Full Configuration with All Options + ```bash # Required export CLAUDE_CODE_ENABLE_TELEMETRY=1 @@ -106,6 +125,7 @@ export OTEL_LOG_USER_PROMPTS=1 ``` ### Remote Server Configuration + ```bash # Required export CLAUDE_CODE_ENABLE_TELEMETRY=1 @@ -119,16 +139,19 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://telemetry.example.com:4318 ## Troubleshooting ### No telemetry received + 1. Check `CLAUDE_CODE_ENABLE_TELEMETRY=1` is set 2. Verify all required exporters are enabled 3. Ensure protocol is `http/json` not `protobuf` 4. Check server is running on the configured endpoint ### Only logs, no metrics + - Set `OTEL_METRICS_EXPORTER=otlp` - Set `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/json` ### Authentication errors + - Check if your server requires API key authentication - Set `API_KEY` environment variable on the server side @@ -137,6 +160,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://telemetry.example.com:4318 To make these settings permanent, add them to your shell configuration: ### Bash (~/.bashrc or ~/.bash_profile) + ```bash # Claude Code Telemetry export CLAUDE_CODE_ENABLE_TELEMETRY=1 @@ -149,6 +173,7 @@ export OTEL_LOG_USER_PROMPTS=1 ``` ### Zsh (~/.zshrc) + ```bash # Claude Code Telemetry export CLAUDE_CODE_ENABLE_TELEMETRY=1 @@ -161,6 +186,7 @@ export OTEL_LOG_USER_PROMPTS=1 ``` ### Fish (~/.config/fish/config.fish) + ```fish # Claude Code Telemetry set -x CLAUDE_CODE_ENABLE_TELEMETRY 1 @@ -171,4 +197,3 @@ set -x OTEL_EXPORTER_OTLP_METRICS_PROTOCOL http/json set -x OTEL_EXPORTER_OTLP_ENDPOINT http://127.0.0.1:4318 set -x OTEL_LOG_USER_PROMPTS 1 ``` - diff --git a/docs/TELEMETRY_GUIDE.md b/docs/TELEMETRY_GUIDE.md index f735133..544ac22 100644 --- a/docs/TELEMETRY_GUIDE.md +++ b/docs/TELEMETRY_GUIDE.md @@ -15,10 +15,12 @@ This guide consolidates all information about Claude Code's telemetry implementa ## Overview Claude Code uses OpenTelemetry (OTLP) to send telemetry data in two forms: + - **Logs**: Events like user prompts, API requests, tool usage - **Metrics**: Numerical data like costs, tokens, code changes The telemetry flows as follows: + ``` Claude Code → OTLP/HTTP → Bridge Server → Langfuse API (JSON logs) (Parse & Map) (Traces/Spans) @@ -64,6 +66,7 @@ Claude sends events as OTLP log records with the event name in the body. ### Event Types #### `claude_code.user_prompt` + - **When**: User sends a prompt to Claude - **Attributes**: - `session.id`: Unique session identifier @@ -72,6 +75,7 @@ Claude sends events as OTLP log records with the event name in the body. - `user.email`: User email if available #### `claude_code.api_request` + - **When**: Claude makes API calls (typically 2 per prompt) - **Models**: `claude-3-haiku` (routing), `claude-3-opus` or `claude-3-5-sonnet` (generation) - **Attributes**: @@ -84,6 +88,7 @@ Claude sends events as OTLP log records with the event name in the body. - `api.response_time`: Alternative duration field #### `claude_code.tool_result` + - **When**: Claude uses tools (Read, Write, Bash, etc.) - **Attributes**: - `tool_name`: Tool name (or `tool` in legacy format) @@ -91,6 +96,7 @@ Claude sends events as OTLP log records with the event name in the body. - `duration_ms`: Execution time in milliseconds #### `claude_code.tool_decision` + - **When**: User makes a decision about tool usage permissions - **Attributes**: - `decision`: User's decision ("accept"/"reject") @@ -98,6 +104,7 @@ Claude sends events as OTLP log records with the event name in the body. - `tool_name`: Tool being decided on #### `claude_code.api_error` + - **When**: API calls fail - **Attributes**: - `model`: Model that failed @@ -111,12 +118,14 @@ Metrics provide numerical measurements of Claude's operation. ### Confirmed Metrics (Received in Testing) #### `claude_code.cost.usage` + - **Type**: Sum - **Unit**: USD - **Purpose**: Track API costs per model - **Example**: Haiku: $0.006, Opus: $0.265 #### `claude_code.token.usage` + - **Type**: Sum - **Unit**: tokens - **Purpose**: Detailed token tracking @@ -125,6 +134,7 @@ Metrics provide numerical measurements of Claude's operation. - `model`: Model name #### `claude_code.lines_of_code.count` + - **Type**: Sum - **Unit**: lines - **Purpose**: Track code modifications @@ -133,6 +143,7 @@ Metrics provide numerical measurements of Claude's operation. - **Triggered**: When modifying files with Edit/Write tools #### `claude_code.commit.count` + - **Type**: Sum - **Unit**: commits - **Purpose**: Track git commits created @@ -151,32 +162,33 @@ These metrics are fully implemented and handled, but occur less frequently: ### Events - 100% Coverage ✅ -| Event | Documentation | Observed | Implemented | -|-------|--------------|----------|-------------| -| `user_prompt` | ✅ Yes | ✅ Yes | ✅ Yes | -| `api_request` | ✅ Yes | ✅ Yes | ✅ Yes | -| `tool_result` | ✅ Yes | ✅ Yes | ✅ Yes | -| `tool_decision` | ✅ Yes | ✅ Yes | ✅ Yes | -| `api_error` | ✅ Yes | ✅ Yes | ✅ Yes | +| Event | Documentation | Observed | Implemented | +| --------------- | ------------- | -------- | ----------- | +| `user_prompt` | ✅ Yes | ✅ Yes | ✅ Yes | +| `api_request` | ✅ Yes | ✅ Yes | ✅ Yes | +| `tool_result` | ✅ Yes | ✅ Yes | ✅ Yes | +| `tool_decision` | ✅ Yes | ✅ Yes | ✅ Yes | +| `api_error` | ✅ Yes | ✅ Yes | ✅ Yes | ### Metrics - 100% Implementation ✅ -| Metric | Purpose | Frequency | Implemented | -|--------|---------|-----------|-------------| -| `cost.usage` | Track API costs | Every API call | ✅ Yes | -| `token.usage` | Track token usage | Every API call | ✅ Yes | -| `lines_of_code.count` | Track code changes | When editing files | ✅ Yes | -| `commit.count` | Track git commits | When committing | ✅ Yes | -| `session.count` | Track new sessions | Session start | ✅ Yes | -| `pull_request.count` | Track PRs created | When creating PRs | ✅ Yes | -| `code_edit_tool.decision` | Track tool permissions | User decisions | ✅ Yes | -| `active_time.total` | Track active time | Periodically | ✅ Yes | +| Metric | Purpose | Frequency | Implemented | +| ------------------------- | ---------------------- | ------------------ | ----------- | +| `cost.usage` | Track API costs | Every API call | ✅ Yes | +| `token.usage` | Track token usage | Every API call | ✅ Yes | +| `lines_of_code.count` | Track code changes | When editing files | ✅ Yes | +| `commit.count` | Track git commits | When committing | ✅ Yes | +| `session.count` | Track new sessions | Session start | ✅ Yes | +| `pull_request.count` | Track PRs created | When creating PRs | ✅ Yes | +| `code_edit_tool.decision` | Track tool permissions | User decisions | ✅ Yes | +| `active_time.total` | Track active time | Periodically | ✅ Yes | ## Real-World Examples ### Typical User Prompt Flow 1. **User Prompt Event** + ```json { "body": "claude_code.user_prompt", @@ -188,6 +200,7 @@ These metrics are fully implemented and handled, but occur less frequently: ``` 2. **Haiku API Request** (routing) + ```json { "body": "claude_code.api_request", @@ -217,17 +230,20 @@ These metrics are fully implemented and handled, but occur less frequently: ### Code Modification Example When Claude modifies a file: + ```json { "name": "claude_code.lines_of_code.count", "sum": { - "dataPoints": [{ - "asDouble": 10, - "attributes": [ - { "key": "type", "value": { "stringValue": "added" } }, - { "key": "session.id", "value": { "stringValue": "abc-123" } } - ] - }] + "dataPoints": [ + { + "asDouble": 10, + "attributes": [ + { "key": "type", "value": { "stringValue": "added" } }, + { "key": "session.id", "value": { "stringValue": "abc-123" } } + ] + } + ] } } ``` @@ -255,6 +271,7 @@ When Claude modifies a file: ### Cost Observations Typical costs per interaction: + - Haiku (routing): $0.001-0.002 - Opus (generation): $0.22-0.28 - Total per prompt: ~$0.23-0.30 @@ -264,4 +281,3 @@ Typical costs per interaction: - Multiple API calls per prompt (typically 2) - Cache usage significantly reduces costs - Tool execution adds to response time - diff --git a/docs/TESTING.md b/docs/TESTING.md index cc6fc70..a8d2f8a 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -5,6 +5,7 @@ This comprehensive guide covers all aspects of testing the Claude Code Telemetry ## Overview The project uses Jest for all testing with three levels: + 1. **Unit Tests** - Test individual modules without external dependencies 2. **Integration Tests** - Test with real server and API interactions 3. **End-to-End Tests** - Test with real Claude binary (requires credentials) @@ -12,17 +13,19 @@ The project uses Jest for all testing with three levels: ## Test Structure ### Unit Tests (`test/unit/`) + - **Purpose**: Test pure functions and logic in isolation - **No Server Required**: Run without external dependencies - **Fast Execution**: Complete in < 1 second -- **Files**: +- **Files**: - `eventProcessor.test.js` - Event handling logic - `metricsProcessor.test.js` - Metrics processing - `sessionHandler.test.js` - Session management (831 lines, most comprehensive) - `serverHelpers.test.js` - Helper functions - `requestHandlers.test.js` - Request processing -### Integration Tests +### Integration Tests + - **Purpose**: Test with real server and Langfuse API - **Auto-starts Server**: Test suite manages server lifecycle - **Real HTTP Requests**: Uses fetch() to test actual endpoints @@ -34,6 +37,7 @@ The project uses Jest for all testing with three levels: - `test/integration/e2e.integration.test.js` - Full E2E with Claude ### Test Helpers + - `testServer.js` - Manages server lifecycle for tests - `helpers/langfuse-client.js` - Direct Langfuse API access - `helpers/otlp-test-data.js` - Consistent test fixtures @@ -41,6 +45,7 @@ The project uses Jest for all testing with three levels: ## Running Tests ### All Tests + ```bash # Run all tests npm test @@ -50,6 +55,7 @@ npm run test:coverage ``` ### Specific Test Types + ```bash # Unit tests only (fast, no dependencies) npm run test:unit @@ -62,6 +68,7 @@ npm run test:ci ``` ### Manual Testing with Real Claude + ```bash # 1. Start the server npm start @@ -81,6 +88,7 @@ claude "What is 2+2?" ## Test Coverage Current coverage: **95.44%** + - Statements: 95.43% - Branches: 77.13% - Functions: 92.45% @@ -109,6 +117,7 @@ Current coverage: **95.44%** ## Writing New Tests ### Unit Test Template + ```javascript // Create in test/unit/myModule.test.js jest.mock('langfuse') // Mock external dependencies @@ -124,28 +133,29 @@ describe('My Module', () => { ``` ### Integration Test Template + ```javascript // Create new file or add to existing integration test const { startTestServer, stopTestServer } = require('./testServer') describe('My Integration Test', () => { let serverProcess, baseUrl - + beforeAll(async () => { const result = await startTestServer('my.test.js') serverProcess = result.serverProcess baseUrl = result.baseUrl }, 15000) - + afterAll(async () => { await stopTestServer(serverProcess) }) - + test('should handle requests', async () => { const response = await fetch(`${baseUrl}/v1/logs`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(testData) + body: JSON.stringify(testData), }) expect(response.status).toBe(200) }) @@ -165,6 +175,7 @@ describe('My Integration Test', () => { ## Troubleshooting ### Integration Tests Fail + ```bash # Kill any lingering processes lsof -ti:4318 | xargs kill -9 2>/dev/null @@ -175,20 +186,24 @@ echo $LANGFUSE_SECRET_KEY ``` ### "Jest did not exit" Warning + - Normal for integration tests - Tests use `--forceExit` flag - Caused by Langfuse SDK intervals ### E2E Tests Skipped + - Requires Claude binary in PATH - Requires Langfuse credentials - Set `CI=false` to enable in CI ### Tests Hang After Completion + - Async operations not properly cleaned up - Ensure all intervals/timeouts are cleared in afterAll hooks ### Debug Mode + ```bash # For tests LOG_LEVEL=debug npm test @@ -200,6 +215,7 @@ LOG_LEVEL=debug npm start ## CI/CD Integration ### GitHub Actions Example + ```yaml name: Telemetry Tests on: [push, pull_request] @@ -209,21 +225,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - + - name: Setup Node.js uses: actions/setup-node@v3 with: node-version: '18' - + - name: Install dependencies run: npm ci - + - name: Run tests with coverage run: npm run test:coverage env: LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} - + - name: Upload coverage reports uses: actions/upload-artifact@v3 if: always() @@ -248,4 +264,4 @@ jobs: ## Summary -The test suite provides comprehensive coverage (95.44%) using Jest for both unit and integration testing. Tests are fast, reliable, and can be run in isolation or as a complete suite. The modular structure makes it easy to add new tests as the codebase evolves. \ No newline at end of file +The test suite provides comprehensive coverage (95.44%) using Jest for both unit and integration testing. Tests are fast, reliable, and can be run in isolation or as a complete suite. The modular structure makes it easy to add new tests as the codebase evolves. diff --git a/jest.config.js b/jest.config.js index 06524cc..3ac00b1 100644 --- a/jest.config.js +++ b/jest.config.js @@ -6,26 +6,21 @@ module.exports = { '!src/**/*.test.js', '!src/server.js', // Tested via integration tests with spawned processes ], - testMatch: [ - '**/test/**/*.test.js', - ], + testMatch: ['**/test/**/*.test.js'], testTimeout: 15000, // Allow time for server startup in integration tests setupFilesAfterEnv: ['/test/setup.js'], // Projects for different test types projects: [ { displayName: 'unit', - testMatch: [ - '/test/helpers.test.js', - '/test/unit/**/*.test.js' - ], + testMatch: ['/test/helpers.test.js', '/test/unit/**/*.test.js'], }, { displayName: 'integration', testMatch: [ '/test/server.test.js', '/test/*.integration.test.js', - '/test/integration/**/*.test.js' + '/test/integration/**/*.test.js', ], maxWorkers: 1, // Run integration tests sequentially to avoid port conflicts testTimeout: 60000, // Longer timeout for real Langfuse calls @@ -37,10 +32,10 @@ module.exports = { // All extracted business logic modules have >85% coverage coverageThreshold: { global: { - branches: 76, // Close to 80%, limited by conditional branches - functions: 86, // Achieved 86.27% - lines: 94, // Achieved 94.36% - statements: 93, // Achieved 93.29% + branches: 76, // Close to 80%, limited by conditional branches + functions: 86, // Achieved 86.27% + lines: 94, // Achieved 94.36% + statements: 93, // Achieved 93.29% }, }, -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 21ff0fb..b23f8b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -77,6 +77,7 @@ "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -1737,6 +1738,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2153,6 +2155,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -2919,6 +2922,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -3106,6 +3110,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -3163,6 +3168,7 @@ "integrity": "sha512-6TyDmZ1HXoFQXnhCTUjVFULReoBPOAjpuiKELMkeP40yffI/1ZRO+d9ug/VC6fqISo2WkuIBk3cvuRPALaWlOQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "builtins": "^5.0.1", @@ -3205,6 +3211,7 @@ "integrity": "sha512-57Zzfw8G6+Gq7axm2Pdo3gW/Rx3h9Yywgn61uE/3elTCOePEHVrn2i5CdfBwA1BLK0Q0WqctICIUSqXZW/VprQ==", "dev": true, "license": "ISC", + "peer": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, diff --git a/package.json b/package.json index 2307eb8..8f2a77a 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,10 @@ "format:check": "prettier --check \"**/*.{js,json,md}\"", "docker:build": "docker build -t claude-code-telemetry .", "docker:run": "docker-compose up", - "clean": "rm -rf test-results test-runs *.log" + "clean": "rm -rf test-results test-runs *.log", + "quality": "bash scripts/ci/run_quality_gates.sh", + "security": "bash scripts/ci/run_security_review.sh", + "hooks:install": "bash scripts/install-hooks.sh" }, "keywords": [ "claude", @@ -56,7 +59,7 @@ "scripts/", "test/fixtures/", "README.md", - "CLAUDE.md", + "AGENTS.md", "LICENSE", ".env.example" ], diff --git a/scripts/ci/run_quality_gates.sh b/scripts/ci/run_quality_gates.sh new file mode 100755 index 0000000..6677441 --- /dev/null +++ b/scripts/ci/run_quality_gates.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Local quality gate — same checks as CI, runnable offline +set -e +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$REPO_ROOT" + +echo "==> format:check" +npm run format:check + +echo "==> lint" +npm run lint + +echo "==> unit tests" +npm run test:unit + +echo "==> quality gate passed" diff --git a/scripts/ci/run_security_review.sh b/scripts/ci/run_security_review.sh new file mode 100755 index 0000000..8aada3e --- /dev/null +++ b/scripts/ci/run_security_review.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Local security review — mirrors CI security job +# NOTE: npm audit depends on the live npm registry (partial enforcement) +set -e +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$REPO_ROOT" + +echo "==> dependency audit (network required)" +npm audit --audit-level=high || echo "WARN: audit found issues — review above output" + +echo "==> secret scan (full tree, excluding node_modules and tests)" +if git grep -E '(api[_-]?key|api[_-]?secret|auth[_-]?token|private[_-]?key|secret[_-]?key)\s*[:=]\s*[A-Za-z0-9+/]{20,}' \ + -- ':(exclude)node_modules/' ':(exclude)*.md' ':(exclude)test/' ':(exclude)*.json' 2>/dev/null; then + echo "ERROR: potential secrets found — review above matches" + exit 1 +fi +echo "No secrets detected" + +echo "==> docker build smoke check (skipped if docker unavailable)" +if command -v docker >/dev/null 2>&1; then + docker build -t claude-telemetry:security-check . --quiet + echo "Docker build OK" +else + echo "SKIP: docker not available locally" +fi + +echo "==> security review complete" +echo "MANUAL REVIEW REQUIRED:" +echo " - auth boundaries in src/server.js (API_KEY middleware)" +echo " - child_process / shell calls (none currently, verify if added)" +echo " - outbound HTTP calls to Langfuse (src/sessionHandler.js)" +echo " - FALLBACK_FILE path traversal (src/offlineFallback.js)" diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh new file mode 100755 index 0000000..f40ed7c --- /dev/null +++ b/scripts/install-hooks.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Wire repo-local git hooks from .githooks/ +set -e +REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" +git -C "$REPO_ROOT" config core.hooksPath .githooks +chmod +x "$REPO_ROOT/.githooks/pre-commit" +echo "Git hooks installed — using .githooks/" diff --git a/src/eventProcessor.js b/src/eventProcessor.js index 8ecdaad..ccfd62c 100644 --- a/src/eventProcessor.js +++ b/src/eventProcessor.js @@ -81,27 +81,36 @@ function processUserPrompt(attrs, standardAttrs, timestamp, session) { const promptLength = parseInt(attrs.prompt_length || attrs['prompt.length'] || '0', 10) const eventTimestamp = attrs['event.timestamp'] || standardAttrs.timestamp - logger.debug({ attrs, prompt, promptLength, standardAttrs }, 'Processing user prompt with attributes') + logger.debug( + { attrs, prompt, promptLength, standardAttrs }, + 'Processing user prompt with attributes', + ) // Pass all attributes to session handler - session.handleUserPrompt({ - prompt, - prompt_length: promptLength, - 'user.email': standardAttrs.userEmail, - 'event.timestamp': eventTimestamp, - 'session.id': standardAttrs.sessionId, - 'organization.id': standardAttrs.organizationId, - 'user.account_uuid': standardAttrs.userAccountUuid, - 'terminal.type': standardAttrs.terminalType, - 'app.version': standardAttrs.appVersion, - }, eventTimestamp) - - logger.info({ - sessionId: session.sessionId, - userEmail: standardAttrs.userEmail, - promptLength, - timestamp: eventTimestamp, - }, 'User prompt received') + session.handleUserPrompt( + { + prompt, + prompt_length: promptLength, + 'user.email': standardAttrs.userEmail, + 'event.timestamp': eventTimestamp, + 'session.id': standardAttrs.sessionId, + 'organization.id': standardAttrs.organizationId, + 'user.account_uuid': standardAttrs.userAccountUuid, + 'terminal.type': standardAttrs.terminalType, + 'app.version': standardAttrs.appVersion, + }, + eventTimestamp, + ) + + logger.info( + { + sessionId: session.sessionId, + userEmail: standardAttrs.userEmail, + promptLength, + timestamp: eventTimestamp, + }, + 'User prompt received', + ) return { type: 'user_prompt', @@ -130,38 +139,47 @@ function processApiRequest(attrs, standardAttrs, timestamp, session) { const inputTokens = parseInt(attrs.input_tokens || attrs['tokens.input'] || '0', 10) const outputTokens = parseInt(attrs.output_tokens || attrs['tokens.output'] || '0', 10) const cacheReadTokens = parseInt(attrs.cache_read_tokens || attrs['cache.read_tokens'] || '0', 10) - const cacheCreationTokens = parseInt(attrs.cache_creation_tokens || attrs['cache.creation_tokens'] || '0', 10) + const cacheCreationTokens = parseInt( + attrs.cache_creation_tokens || attrs['cache.creation_tokens'] || '0', + 10, + ) const costUsd = parseFloat(attrs.cost_usd || attrs.cost || attrs['cost.usd'] || '0') const durationMs = parseInt(attrs.duration_ms || attrs.duration || '0', 10) const requestId = attrs.request_id || attrs['request.id'] const eventTimestamp = attrs['event.timestamp'] || standardAttrs.timestamp // Pass all attributes to session handler - session.handleApiRequest({ - model, - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_read_tokens: cacheReadTokens, - cache_creation_tokens: cacheCreationTokens, - cost: costUsd, - duration_ms: durationMs, - 'api.response_time': durationMs, - request_id: requestId, - 'event.timestamp': eventTimestamp, - ...standardAttrs, - }, eventTimestamp) - - logger.info({ - sessionId: session.sessionId, - model, - inputTokens, - outputTokens, - cacheReadTokens, - cacheCreationTokens, - costUsd, - durationMs, - requestId, - }, 'API request processed') + session.handleApiRequest( + { + model, + input_tokens: inputTokens, + output_tokens: outputTokens, + cache_read_tokens: cacheReadTokens, + cache_creation_tokens: cacheCreationTokens, + cost: costUsd, + duration_ms: durationMs, + 'api.response_time': durationMs, + request_id: requestId, + 'event.timestamp': eventTimestamp, + ...standardAttrs, + }, + eventTimestamp, + ) + + logger.info( + { + sessionId: session.sessionId, + model, + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens, + costUsd, + durationMs, + requestId, + }, + 'API request processed', + ) return { type: 'api_request', @@ -194,22 +212,28 @@ function processApiError(attrs, standardAttrs, timestamp, session) { const requestId = attrs.request_id || attrs['request.id'] const eventTimestamp = attrs['event.timestamp'] || standardAttrs.timestamp - session.handleApiError({ - error_message: errorMessage, - status_code: statusCode, - model, - request_id: requestId, - 'event.timestamp': eventTimestamp, - ...standardAttrs, - }, eventTimestamp) - - logger.warn({ - sessionId: session.sessionId, - errorMessage, - statusCode, - model, - requestId, - }, 'API error occurred') + session.handleApiError( + { + error_message: errorMessage, + status_code: statusCode, + model, + request_id: requestId, + 'event.timestamp': eventTimestamp, + ...standardAttrs, + }, + eventTimestamp, + ) + + logger.warn( + { + sessionId: session.sessionId, + errorMessage, + statusCode, + model, + requestId, + }, + 'API error occurred', + ) return { type: 'api_error', @@ -236,20 +260,26 @@ function processToolResult(attrs, standardAttrs, timestamp, session) { const durationMs = parseInt(attrs.duration_ms || attrs.duration || '0', 10) const eventTimestamp = attrs['event.timestamp'] || standardAttrs.timestamp - session.handleToolResult({ - tool_name: toolName, - success, - duration_ms: durationMs, - 'event.timestamp': eventTimestamp, - ...standardAttrs, - }, eventTimestamp) - - logger.info({ - sessionId: session.sessionId, - toolName, - success, - durationMs, - }, 'Tool result processed') + session.handleToolResult( + { + tool_name: toolName, + success, + duration_ms: durationMs, + 'event.timestamp': eventTimestamp, + ...standardAttrs, + }, + eventTimestamp, + ) + + logger.info( + { + sessionId: session.sessionId, + toolName, + success, + durationMs, + }, + 'Tool result processed', + ) return { type: 'tool_result', @@ -276,12 +306,15 @@ function processToolDecision(attrs, standardAttrs, timestamp, session) { const toolName = attrs.tool_name || attrs.tool || 'unknown' const eventTimestamp = attrs['event.timestamp'] || standardAttrs.timestamp - logger.info({ - sessionId: session.sessionId, - decision, - source, - toolName, - }, 'Tool decision processed') + logger.info( + { + sessionId: session.sessionId, + decision, + source, + toolName, + }, + 'Tool decision processed', + ) // Create event in current trace if exists if (session.currentTrace && session.langfuse) { diff --git a/src/metricsProcessor.js b/src/metricsProcessor.js index 4824bb3..585b6eb 100644 --- a/src/metricsProcessor.js +++ b/src/metricsProcessor.js @@ -63,12 +63,15 @@ function processCostMetric(dataPoint, attrs, timestamp, session) { const model = attrs.model || 'unknown' // Cost is now handled by session.processMetric() - logger.info({ - sessionId: session.sessionId, - cost, - model, - totalCost: session.totalCost, - }, 'Cost metric processed') + logger.info( + { + sessionId: session.sessionId, + cost, + model, + totalCost: session.totalCost, + }, + 'Cost metric processed', + ) return { type: 'cost', @@ -87,12 +90,15 @@ function processTokenMetric(dataPoint, attrs, timestamp, session) { const model = attrs.model || 'unknown' // Tokens are now handled by session.processMetric() - logger.info({ - sessionId: session.sessionId, - tokens, - tokenType, - model, - }, 'Token metric processed') + logger.info( + { + sessionId: session.sessionId, + tokens, + tokenType, + model, + }, + 'Token metric processed', + ) return { type: 'token', @@ -111,13 +117,16 @@ function processLinesOfCodeMetric(dataPoint, attrs, timestamp, session) { const changeType = attrs.type || 'unknown' // Lines tracking is now handled by session.processMetric() - logger.info({ - sessionId: session.sessionId, - lines, - changeType, - totalAdded: session.linesAdded, - totalRemoved: session.linesRemoved, - }, 'Lines of code metric processed') + logger.info( + { + sessionId: session.sessionId, + lines, + changeType, + totalAdded: session.linesAdded, + totalRemoved: session.linesRemoved, + }, + 'Lines of code metric processed', + ) return { type: 'lines_of_code', @@ -139,11 +148,14 @@ function processCommitMetric(dataPoint, attrs, timestamp, session) { } session.commitCount += commits - logger.info({ - sessionId: session.sessionId, - commits, - totalCommits: session.commitCount, - }, 'Commit metric processed') + logger.info( + { + sessionId: session.sessionId, + commits, + totalCommits: session.commitCount, + }, + 'Commit metric processed', + ) return { type: 'commit', @@ -202,13 +214,16 @@ function processToolDecisionMetric(dataPoint, attrs, timestamp, session) { }) } - logger.info({ - sessionId: session.sessionId, - decision, - tool, - language, - count, - }, 'Tool decision metric processed') + logger.info( + { + sessionId: session.sessionId, + decision, + tool, + language, + count, + }, + 'Tool decision metric processed', + ) return { type: 'tool_decision', @@ -227,10 +242,13 @@ function processSessionMetric(dataPoint, attrs, timestamp, session) { // This metric indicates a new session started session.sessionStarted = true - logger.info({ - sessionId: session.sessionId, - timestamp: new Date(timestamp).toISOString(), - }, 'Session start metric processed') + logger.info( + { + sessionId: session.sessionId, + timestamp: new Date(timestamp).toISOString(), + }, + 'Session start metric processed', + ) return { type: 'session_start', @@ -246,10 +264,13 @@ function processActiveTimeMetric(dataPoint, attrs, timestamp, session) { session.activeTime = activeTime - logger.info({ - sessionId: session.sessionId, - activeTime, - }, 'Active time metric processed') + logger.info( + { + sessionId: session.sessionId, + activeTime, + }, + 'Active time metric processed', + ) return { type: 'active_time', @@ -266,11 +287,14 @@ function processPullRequestMetric(dataPoint, attrs, timestamp, session) { session.prCount = (session.prCount || 0) + prCount - logger.info({ - sessionId: session.sessionId, - prCount, - totalPRs: session.prCount, - }, 'Pull request metric processed') + logger.info( + { + sessionId: session.sessionId, + prCount, + totalPRs: session.prCount, + }, + 'Pull request metric processed', + ) return { type: 'pull_request', diff --git a/src/offlineFallback.js b/src/offlineFallback.js new file mode 100644 index 0000000..d050f91 --- /dev/null +++ b/src/offlineFallback.js @@ -0,0 +1,32 @@ +const fs = require('fs') +const path = require('path') + +const DEFAULT_PATH = path.join(process.env.HOME || '/tmp', '.claude', 'telemetry-fallback.jsonl') + +function getFallbackPath() { + return process.env.FALLBACK_FILE || DEFAULT_PATH +} + +function writeReference(session) { + const ref = { + sessionId: session.sessionId, + userId: session.metadata?.userId, + organizationId: session.organizationId, + userAccountUuid: session.userAccountUuid, + createdAt: session.createdAt?.toISOString(), + failedAt: new Date().toISOString(), + stats: { + conversationCount: session.conversationCount, + apiCallCount: session.apiCallCount, + totalTokens: session.totalTokens, + totalCost: session.totalCost, + }, + } + + const dir = path.dirname(getFallbackPath()) + fs.mkdirSync(dir, { recursive: true }) + fs.appendFileSync(getFallbackPath(), JSON.stringify(ref) + '\n') + return ref +} + +module.exports = { writeReference, getFallbackPath } diff --git a/src/requestHandlers.js b/src/requestHandlers.js index b8dcf38..4e809cc 100644 --- a/src/requestHandlers.js +++ b/src/requestHandlers.js @@ -48,17 +48,26 @@ function handleMetrics(data, res, sessions, langfuse) { for (const scopeMetric of resourceMetric.scopeMetrics || []) { for (const metric of scopeMetric.metrics || []) { - logger.info({ - metricName: metric.name, - metricDescription: metric.description, - metricUnit: metric.unit, - dataPoints: metric.sum?.dataPoints?.length || - metric.gauge?.dataPoints?.length || - metric.histogram?.dataPoints?.length || 0, - }, 'Processing metric') + logger.info( + { + metricName: metric.name, + metricDescription: metric.description, + metricUnit: metric.unit, + dataPoints: + metric.sum?.dataPoints?.length || + metric.gauge?.dataPoints?.length || + metric.histogram?.dataPoints?.length || + 0, + }, + 'Processing metric', + ) // Process metric data points - const dataPoints = metric.sum?.dataPoints || metric.gauge?.dataPoints || metric.histogram?.dataPoints || [] + const dataPoints = + metric.sum?.dataPoints || + metric.gauge?.dataPoints || + metric.histogram?.dataPoints || + [] for (const dataPoint of dataPoints) { const attrs = extractAttributesArray(dataPoint.attributes) diff --git a/src/server.js b/src/server.js index 6e63812..b94bff8 100644 --- a/src/server.js +++ b/src/server.js @@ -42,16 +42,17 @@ const config = createConfig() // Logger setup const logger = pino({ level: config.logLevel, - transport: config.nodeEnv === 'development' - ? { - target: 'pino-pretty', - options: { - translateTime: 'HH:MM:ss.l', - ignore: 'pid,hostname', - colorize: true, - }, - } - : undefined, + transport: + config.nodeEnv === 'development' + ? { + target: 'pino-pretty', + options: { + translateTime: 'HH:MM:ss.l', + ignore: 'pid,hostname', + colorize: true, + }, + } + : undefined, }) // Validate configuration diff --git a/src/serverHelpers.js b/src/serverHelpers.js index 2eede73..00c049f 100644 --- a/src/serverHelpers.js +++ b/src/serverHelpers.js @@ -50,6 +50,7 @@ function createConfig() { apiKey: process.env.API_KEY, nodeEnv: process.env.NODE_ENV || 'production', logLevel: process.env.LOG_LEVEL || 'info', + fallbackFile: process.env.FALLBACK_FILE || '', } } diff --git a/src/sessionHandler.js b/src/sessionHandler.js index c904e43..ef363ec 100644 --- a/src/sessionHandler.js +++ b/src/sessionHandler.js @@ -1,19 +1,21 @@ // Session Handler class extracted for better testability // const { Langfuse } = require('langfuse') // Imported but not used directly in this file const pino = require('pino') +const { writeReference, getFallbackPath } = require('./offlineFallback') const logger = pino({ level: process.env.LOG_LEVEL || 'info', - transport: process.env.NODE_ENV === 'development' - ? { - target: 'pino-pretty', - options: { - translateTime: 'HH:MM:ss.l', - ignore: 'pid,hostname', - colorize: true, - }, - } - : undefined, + transport: + process.env.NODE_ENV === 'development' + ? { + target: 'pino-pretty', + options: { + translateTime: 'HH:MM:ss.l', + ignore: 'pid,hostname', + colorize: true, + }, + } + : undefined, }) class SessionHandler { @@ -76,7 +78,8 @@ class SessionHandler { const attrs = resourceAttributes || {} return { name: attrs['service.name'] || 'claude-code', - version: attrs['service.version'] || attrs['claude.version'] || attrs['app.version'] || 'unknown', + version: + attrs['service.version'] || attrs['claude.version'] || attrs['app.version'] || 'unknown', instance: attrs['service.instance.id'] || attrs['host.name'], telemetrySDK: attrs['telemetry.sdk.name'], terminalType: attrs['terminal.type'], @@ -85,7 +88,9 @@ class SessionHandler { processLogRecord(logRecord, resource) { const eventName = logRecord.body?.stringValue - const timestamp = logRecord.timeUnixNano ? new Date(Number(logRecord.timeUnixNano) / 1000000).toISOString() : new Date().toISOString() + const timestamp = logRecord.timeUnixNano + ? new Date(Number(logRecord.timeUnixNano) / 1000000).toISOString() + : new Date().toISOString() const attrs = extractAttributesArray(logRecord.attributes) logger.debug({ eventName, attrs, sessionId: this.sessionId }, 'Processing event') @@ -154,7 +159,9 @@ class SessionHandler { const requestId = attrs.request_id const startTime = new Date(timestamp) - const endTime = attrs['api.response_time'] ? new Date(new Date(timestamp).getTime() + (attrs['api.response_time'] || 0)) : new Date() + const endTime = attrs['api.response_time'] + ? new Date(new Date(timestamp).getTime() + (attrs['api.response_time'] || 0)) + : new Date() const durationMs = attrs['api.response_time'] || attrs.duration || endTime - startTime // Update metrics @@ -194,14 +201,17 @@ class SessionHandler { // Create generation span const modelType = model.includes('haiku') ? 'routing' : 'generation' - logger.debug({ - sessionId: this.sessionId, - traceId: this.currentTrace?.id, - model, - modelType, - hasTrace: !!this.currentTrace, - langfuseAvailable: !!this.langfuse, - }, 'Creating generation observation') + logger.debug( + { + sessionId: this.sessionId, + traceId: this.currentTrace?.id, + model, + modelType, + hasTrace: !!this.currentTrace, + langfuseAvailable: !!this.langfuse, + }, + 'Creating generation observation', + ) const span = this.langfuse.generation({ name: `${modelType}-${model}`, @@ -244,11 +254,14 @@ class SessionHandler { version: this.metadata.release, }) - logger.debug({ - sessionId: this.sessionId, - generationId: span?.id, - modelType, - }, 'Generation observation created') + logger.debug( + { + sessionId: this.sessionId, + generationId: span?.id, + modelType, + }, + 'Generation observation created', + ) if (modelType === 'generation') { this.currentSpan = span @@ -281,7 +294,8 @@ class SessionHandler { const source = attrs.source || 'automated' this.toolCallCount++ - const startTime = durationMs > 0 ? new Date(new Date(timestamp).getTime() - durationMs) : new Date(timestamp) + const startTime = + durationMs > 0 ? new Date(new Date(timestamp).getTime() - durationMs) : new Date(timestamp) // Track tool sequence this.toolSequence.push({ @@ -341,13 +355,16 @@ class SessionHandler { const errorMessage = attrs.error_message || attrs.error || 'Unknown error' const statusCode = attrs.status_code || attrs.status || 0 - logger.error({ - sessionId: this.sessionId, - model, - error: errorMessage, - statusCode, - timestamp, - }, 'API error occurred') + logger.error( + { + sessionId: this.sessionId, + model, + error: errorMessage, + statusCode, + timestamp, + }, + 'API error occurred', + ) if (this.currentTrace) { this.langfuse.event({ @@ -372,10 +389,13 @@ class SessionHandler { case 'claude_code.session.count': { // Session count metric const sessionCount = dataPoint.asDouble || dataPoint.asInt || 1 - logger.info({ - sessionId: this.sessionId, - count: sessionCount, - }, 'Session count metric') + logger.info( + { + sessionId: this.sessionId, + count: sessionCount, + }, + 'Session count metric', + ) // Create a session started event if (this.currentTrace) { @@ -397,7 +417,10 @@ class SessionHandler { const cost = dataPoint.asDouble || 0 const costModel = attrs.model || 'unknown' this.totalCost += cost - logger.info({ sessionId: this.sessionId, cost, model: costModel, totalCost: this.totalCost }, 'Cost metric received') + logger.info( + { sessionId: this.sessionId, cost, model: costModel, totalCost: this.totalCost }, + 'Cost metric received', + ) break } @@ -425,14 +448,17 @@ class SessionHandler { break } - logger.info({ - sessionId: this.sessionId, - tokens, - tokenType, - model: tokenModel, - totalTokens: this.totalTokens, - cacheTokens: this.cacheTokens, - }, 'Token metric received') + logger.info( + { + sessionId: this.sessionId, + tokens, + tokenType, + model: tokenModel, + totalTokens: this.totalTokens, + cacheTokens: this.cacheTokens, + }, + 'Token metric received', + ) break } @@ -461,21 +487,27 @@ class SessionHandler { }) } - logger.info({ - sessionId: this.sessionId, - lines, - type: modificationType, - }, 'Code modification metric') + logger.info( + { + sessionId: this.sessionId, + lines, + type: modificationType, + }, + 'Code modification metric', + ) break } case 'claude_code.pull_request.count': { // PR creation metric const prCount = dataPoint.asDouble || dataPoint.asInt || 1 - logger.info({ - sessionId: this.sessionId, - count: prCount, - }, 'Pull request created') + logger.info( + { + sessionId: this.sessionId, + count: prCount, + }, + 'Pull request created', + ) if (this.currentTrace) { this.langfuse.event({ @@ -494,10 +526,13 @@ class SessionHandler { case 'claude_code.commit.count': { // Git commit metric const commitCount = dataPoint.asDouble || dataPoint.asInt || 1 - logger.info({ - sessionId: this.sessionId, - count: commitCount, - }, 'Git commit created') + logger.info( + { + sessionId: this.sessionId, + count: commitCount, + }, + 'Git commit created', + ) if (this.currentTrace) { this.langfuse.event({ @@ -519,12 +554,15 @@ class SessionHandler { const tool = attrs.tool const language = attrs.language - logger.info({ - sessionId: this.sessionId, - tool, - decision, - language, - }, 'Tool permission decision') + logger.info( + { + sessionId: this.sessionId, + tool, + decision, + language, + }, + 'Tool permission decision', + ) if (this.currentTrace) { this.langfuse.event({ @@ -545,10 +583,13 @@ class SessionHandler { case 'claude_code.active_time.total': { // Active time metric const activeTime = dataPoint.asDouble || 0 - logger.info({ - sessionId: this.sessionId, - activeTimeSeconds: activeTime, - }, 'Active time metric') + logger.info( + { + sessionId: this.sessionId, + activeTimeSeconds: activeTime, + }, + 'Active time metric', + ) if (this.currentTrace) { this.langfuse.event({ @@ -565,12 +606,15 @@ class SessionHandler { } default: - logger.debug({ - sessionId: this.sessionId, - metricName: metric.name, - value: dataPoint.asDouble || dataPoint.asInt || 0, - attributes: attrs, - }, 'Unknown metric received') + logger.debug( + { + sessionId: this.sessionId, + metricName: metric.name, + value: dataPoint.asDouble || dataPoint.asInt || 0, + attributes: attrs, + }, + 'Unknown metric received', + ) } } @@ -672,10 +716,19 @@ class SessionHandler { }) // Calculate quality score - const cacheHitRate = this.totalTokens > 0 ? this.latencies.api.reduce((sum, l) => sum + l, 0) / this.totalTokens : 0 + const cacheHitRate = + this.totalTokens > 0 + ? this.latencies.api.reduce((sum, l) => sum + l, 0) / this.totalTokens + : 0 const avgResponseTime = apiPercentiles ? apiPercentiles.avg : 0 - const toolSuccessRate = this.toolSequence.length > 0 ? this.toolSequence.filter((t) => t.success).length / this.toolSequence.length : 1 - const qualityScore = Math.min(100, Math.round((cacheHitRate * 20) + (toolSuccessRate * 40) + (avgResponseTime < 1000 ? 40 : 20))) + const toolSuccessRate = + this.toolSequence.length > 0 + ? this.toolSequence.filter((t) => t.success).length / this.toolSequence.length + : 1 + const qualityScore = Math.min( + 100, + Math.round(cacheHitRate * 20 + toolSuccessRate * 40 + (avgResponseTime < 1000 ? 40 : 20)), + ) this.langfuse.score({ traceId: sessionSummary.id, @@ -696,12 +749,27 @@ class SessionHandler { } logger.info({ sessionId: this.sessionId, totalCost: this.totalCost }, 'Session finalized') - const baseUrl = (process.env.LANGFUSE_HOST || 'http://localhost:3000').replace(/\/api\/public.*$/, '') + const baseUrl = (process.env.LANGFUSE_HOST || 'http://localhost:3000').replace( + /\/api\/public.*$/, + '', + ) logger.info(`View at: ${baseUrl}/sessions/${this.sessionId}`) await retry(() => this.langfuse.flushAsync()) } catch (error) { logger.error({ error, sessionId: this.sessionId }, 'Error finalizing session') + try { + writeReference(this) + logger.info( + { sessionId: this.sessionId, fallbackPath: getFallbackPath() }, + 'Session reference saved to offline fallback', + ) + } catch (fallbackError) { + logger.error( + { fallbackError, sessionId: this.sessionId }, + 'Failed to write offline fallback', + ) + } } } } diff --git a/test/apiError.integration.test.js b/test/apiError.integration.test.js index 9f76f1b..289748c 100644 --- a/test/apiError.integration.test.js +++ b/test/apiError.integration.test.js @@ -17,26 +17,30 @@ describe('API Error Integration Tests', () => { describe('API Error Events from Logs', () => { test('processes API error with 401 status', async () => { const logData = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'claude-code-telemetry' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.api_error' }, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-api-error-401' } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + { key: 'error_message', value: { stringValue: 'Authentication failed' } }, + { key: 'status_code', value: { intValue: '401' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'claude-code-telemetry' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.api_error' }, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-api-error-401' } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - { key: 'error_message', value: { stringValue: 'Authentication failed' } }, - { key: 'status_code', value: { intValue: '401' } }, - ], - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -52,26 +56,30 @@ describe('API Error Integration Tests', () => { test('processes API error with rate limit', async () => { const logData = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'claude-code-telemetry' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.api_error' }, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-api-error-429' } }, + { key: 'model', value: { stringValue: 'claude-3-haiku' } }, + { key: 'error', value: { stringValue: 'Rate limit exceeded' } }, + { key: 'status', value: { intValue: '429' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'claude-code-telemetry' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.api_error' }, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-api-error-429' } }, - { key: 'model', value: { stringValue: 'claude-3-haiku' } }, - { key: 'error', value: { stringValue: 'Rate limit exceeded' } }, - { key: 'status', value: { intValue: '429' } }, - ], - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -85,24 +93,28 @@ describe('API Error Integration Tests', () => { test('processes API error with network failure', async () => { const logData = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'claude-code-telemetry' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.api_error' }, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-api-error-network' } }, + { key: 'error_message', value: { stringValue: 'Network timeout' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'claude-code-telemetry' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.api_error' }, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-api-error-network' } }, - { key: 'error_message', value: { stringValue: 'Network timeout' } }, - ], - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -121,26 +133,30 @@ describe('API Error Integration Tests', () => { // First, send an API error const errorLog = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'claude-code-telemetry' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.api_error' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + { key: 'error_message', value: { stringValue: 'Temporary failure' } }, + { key: 'status_code', value: { intValue: '503' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'claude-code-telemetry' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.api_error' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - { key: 'error_message', value: { stringValue: 'Temporary failure' } }, - { key: 'status_code', value: { intValue: '503' } }, - ], - }], - }], - }], + ], } // Send error with retry logic @@ -155,7 +171,7 @@ describe('API Error Integration Tests', () => { break } catch (error) { if (retry === 2) throw error - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) } } @@ -163,27 +179,31 @@ describe('API Error Integration Tests', () => { // Then send a successful API request const successLog = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'claude-code-telemetry' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000 + 1000000, + body: { stringValue: 'claude_code.api_request' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + { key: 'input_tokens', value: { intValue: '500' } }, + { key: 'output_tokens', value: { intValue: '1000' } }, + { key: 'cost', value: { doubleValue: 0.25 } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'claude-code-telemetry' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000 + 1000000, - body: { stringValue: 'claude_code.api_request' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - { key: 'input_tokens', value: { intValue: '500' } }, - { key: 'output_tokens', value: { intValue: '1000' } }, - { key: 'cost', value: { doubleValue: 0.25 } }, - ], - }], - }], - }], + ], } const successResponse = await fetch(`${baseUrl}/v1/logs`, { diff --git a/test/fixtures/README.md b/test/fixtures/README.md index 8495edd..47ed58e 100644 --- a/test/fixtures/README.md +++ b/test/fixtures/README.md @@ -11,6 +11,7 @@ This directory contains test fixtures for validating Claude Code telemetry with ## Usage These fixtures are used by: + 1. `scripts/manual-test.sh` - Manual test protocol 2. `scripts/automated-test.sh` - Automated test runner -3. Unit tests in `test/server.test.js` \ No newline at end of file +3. Unit tests in `test/server.test.js` diff --git a/test/fixtures/expected/api-request-event.json b/test/fixtures/expected/api-request-event.json index d32f5f4..8ccd547 100644 --- a/test/fixtures/expected/api-request-event.json +++ b/test/fixtures/expected/api-request-event.json @@ -30,4 +30,4 @@ "pattern": "^[0-9]+$" } } -} \ No newline at end of file +} diff --git a/test/fixtures/expected/tool-result-event.json b/test/fixtures/expected/tool-result-event.json index 43ee2a5..300059f 100644 --- a/test/fixtures/expected/tool-result-event.json +++ b/test/fixtures/expected/tool-result-event.json @@ -1,11 +1,6 @@ { "event_type": "claude_code.tool_result", - "required_attributes": [ - "session.id", - "tool_name", - "success", - "duration_ms" - ], + "required_attributes": ["session.id", "tool_name", "success", "duration_ms"], "validation": { "tool_name": { "type": "string", @@ -20,4 +15,4 @@ "pattern": "^[0-9]+$" } } -} \ No newline at end of file +} diff --git a/test/fixtures/expected/user-prompt-event.json b/test/fixtures/expected/user-prompt-event.json index f325e85..14c1b0e 100644 --- a/test/fixtures/expected/user-prompt-event.json +++ b/test/fixtures/expected/user-prompt-event.json @@ -1,11 +1,6 @@ { "event_type": "claude_code.user_prompt", - "required_attributes": [ - "session.id", - "user.email", - "prompt", - "prompt_length" - ], + "required_attributes": ["session.id", "user.email", "prompt", "prompt_length"], "validation": { "session.id": { "type": "string", @@ -20,4 +15,4 @@ "pattern": "^[0-9]+$" } } -} \ No newline at end of file +} diff --git a/test/helpers.test.js b/test/helpers.test.js index 992ae66..e7898de 100644 --- a/test/helpers.test.js +++ b/test/helpers.test.js @@ -12,7 +12,7 @@ describe('Helper Functions', () => { if (value.doubleValue !== undefined) return parseFloat(value.doubleValue) if (value.boolValue !== undefined) return value.boolValue if (value.arrayValue !== undefined) { - return value.arrayValue.values.map(v => extractAttributeValue(v)) + return value.arrayValue.values.map((v) => extractAttributeValue(v)) } if (value.kvlistValue !== undefined) { const obj = {} @@ -47,11 +47,7 @@ describe('Helper Functions', () => { test('extracts array values', () => { const value = { arrayValue: { - values: [ - { stringValue: 'a' }, - { stringValue: 'b' }, - { intValue: '1' }, - ], + values: [{ stringValue: 'a' }, { stringValue: 'b' }, { intValue: '1' }], }, } expect(extractAttributeValue(value)).toEqual(['a', 'b', 1]) @@ -152,8 +148,12 @@ describe('Helper Functions', () => { } expect(validateConfig({ publicKey: 'pk-test', secretKey: 'sk-test' })).toEqual([]) - expect(validateConfig({ publicKey: '', secretKey: 'sk-test' })).toContain('LANGFUSE_PUBLIC_KEY is required') - expect(validateConfig({ publicKey: 'pk-test', secretKey: '' })).toContain('LANGFUSE_SECRET_KEY is required') + expect(validateConfig({ publicKey: '', secretKey: 'sk-test' })).toContain( + 'LANGFUSE_PUBLIC_KEY is required', + ) + expect(validateConfig({ publicKey: 'pk-test', secretKey: '' })).toContain( + 'LANGFUSE_SECRET_KEY is required', + ) }) }) diff --git a/test/helpers/langfuse-client.js b/test/helpers/langfuse-client.js index 16b6ae8..bb62b43 100644 --- a/test/helpers/langfuse-client.js +++ b/test/helpers/langfuse-client.js @@ -41,7 +41,7 @@ class LangfuseTestClient { const req = client.request(options, (res) => { let data = '' - res.on('data', chunk => { + res.on('data', (chunk) => { data += chunk }) res.on('end', () => { @@ -120,14 +120,16 @@ class LangfuseTestClient { while (Date.now() - startTime < timeout) { const traces = await this.getTraces(20) - const found = traces.find(trace => { + const found = traces.find((trace) => { if (criteria.name && !trace.name?.includes(criteria.name)) return false if (criteria.sessionId && trace.sessionId !== criteria.sessionId) return false if (criteria.hasObservations) { // Need to fetch full trace to check observations - return this.getTrace(trace.id).then(fullTrace => { - return (fullTrace.observations?.length || 0) > 0 - }).catch(() => false) + return this.getTrace(trace.id) + .then((fullTrace) => { + return (fullTrace.observations?.length || 0) > 0 + }) + .catch(() => false) } return true }) @@ -140,7 +142,7 @@ class LangfuseTestClient { return found } - await new Promise(resolve => setTimeout(resolve, interval)) + await new Promise((resolve) => setTimeout(resolve, interval)) } throw new Error(`Trace not found after ${timeout}ms with criteria: ${JSON.stringify(criteria)}`) @@ -177,13 +179,13 @@ class LangfuseTestClient { // Check session exists const sessions = await this.getSessions() - validation.session = sessions.some(s => s.id === sessionId) + validation.session = sessions.some((s) => s.id === sessionId) // Get traces for session const traces = await this.getTraces(20, sessionId) // Check for conversation trace - const conversationTrace = traces.find(t => t.name?.startsWith('conversation-')) + const conversationTrace = traces.find((t) => t.name?.startsWith('conversation-')) if (conversationTrace) { validation.conversationTrace = true @@ -198,11 +200,11 @@ class LangfuseTestClient { // Count observations const observations = fullTrace.observations || [] - validation.generations = observations.filter(o => o.type === 'GENERATION').length - validation.events = observations.filter(o => o.type === 'EVENT').length + validation.generations = observations.filter((o) => o.type === 'GENERATION').length + validation.events = observations.filter((o) => o.type === 'EVENT').length // Check for cache data in generations - const generation = observations.find(o => o.type === 'GENERATION') + const generation = observations.find((o) => o.type === 'GENERATION') if (generation?.metadata?.cache) { validation.metadata.cache = true } @@ -212,7 +214,7 @@ class LangfuseTestClient { } // Check for session summary trace - validation.summaryTrace = traces.some(t => t.name === 'session-summary') + validation.summaryTrace = traces.some((t) => t.name === 'session-summary') return validation } diff --git a/test/helpers/otlp-test-data.js b/test/helpers/otlp-test-data.js index 1f72ff8..069a125 100644 --- a/test/helpers/otlp-test-data.js +++ b/test/helpers/otlp-test-data.js @@ -147,18 +147,20 @@ function createCostMetric(sessionId, model = 'claude-3-opus-20240229', cost = 0. sum: { aggregationTemporality: 1, isMonotonic: true, - dataPoints: [{ - attributes: [ - ...Object.entries(attrs).map(([k, v]) => ({ - key: k, - value: { stringValue: String(v) }, - })), - { key: 'model', value: { stringValue: model } }, - ], - startTimeUnixNano: String(timestamp - 1000000), - timeUnixNano: String(timestamp), - asDouble: cost, - }], + dataPoints: [ + { + attributes: [ + ...Object.entries(attrs).map(([k, v]) => ({ + key: k, + value: { stringValue: String(v) }, + })), + { key: 'model', value: { stringValue: model } }, + ], + startTimeUnixNano: String(timestamp - 1000000), + timeUnixNano: String(timestamp), + asDouble: cost, + }, + ], }, } } @@ -167,12 +169,7 @@ function createCostMetric(sessionId, model = 'claude-3-opus-20240229', cost = 0. * Create token usage metric data points */ function createTokenMetrics(sessionId, model = 'claude-3-opus-20240229', tokens = {}) { - const { - input = 100, - output = 200, - cacheRead = 50, - cacheCreation = 25, - } = tokens + const { input = 100, output = 200, cacheRead = 50, cacheCreation = 25 } = tokens const attrs = createStandardAttributes(sessionId) const timestamp = Date.now() * 1000000 @@ -213,18 +210,22 @@ function createTokenMetrics(sessionId, model = 'claude-3-opus-20240229', tokens */ function createOTLPLogsRequest(logRecords) { return { - resourceLogs: [{ - resource: { - attributes: createResourceAttributes(), - }, - scopeLogs: [{ - scope: { - name: 'com.anthropic.claude_code', - version: '1.0.64', + resourceLogs: [ + { + resource: { + attributes: createResourceAttributes(), }, - logRecords, - }], - }], + scopeLogs: [ + { + scope: { + name: 'com.anthropic.claude_code', + version: '1.0.64', + }, + logRecords, + }, + ], + }, + ], } } @@ -233,18 +234,22 @@ function createOTLPLogsRequest(logRecords) { */ function createOTLPMetricsRequest(metrics) { return { - resourceMetrics: [{ - resource: { - attributes: createResourceAttributes(), - }, - scopeMetrics: [{ - scope: { - name: 'com.anthropic.claude_code', - version: '1.0.64', + resourceMetrics: [ + { + resource: { + attributes: createResourceAttributes(), }, - metrics, - }], - }], + scopeMetrics: [ + { + scope: { + name: 'com.anthropic.claude_code', + version: '1.0.64', + }, + metrics, + }, + ], + }, + ], } } diff --git a/test/helpers/validate-langfuse.js b/test/helpers/validate-langfuse.js index 0d29c7f..f1e38b1 100755 --- a/test/helpers/validate-langfuse.js +++ b/test/helpers/validate-langfuse.js @@ -38,7 +38,7 @@ function langfuseRequest(path, method = 'GET') { const req = client.request(options, (res) => { let data = '' - res.on('data', chunk => { + res.on('data', (chunk) => { data += chunk }) res.on('end', () => { @@ -99,11 +99,11 @@ async function validateComprehensive() { if (traces.data?.length > 0) { // Check for conversation traces - const conversationTrace = traces.data.find(t => t.name?.startsWith('conversation-')) + const conversationTrace = traces.data.find((t) => t.name?.startsWith('conversation-')) validationResults.traces.conversation = !!conversationTrace // Check for session-summary trace - const summaryTrace = traces.data.find(t => t.name === 'session-summary') + const summaryTrace = traces.data.find((t) => t.name === 'session-summary') validationResults.traces.sessionSummary = !!summaryTrace console.log(` ✅ Found ${traces.data.length} traces`) @@ -121,22 +121,30 @@ async function validateComprehensive() { validationResults.metadata.user = !!(metadata.userAccountUuid || metadata.userEmail) validationResults.metadata.terminal = !!metadata.terminalType - console.log(` ${metadata.organizationId ? '✅' : '❌'} Organization ID: ${metadata.organizationId || 'missing'}`) - console.log(` ${metadata.userAccountUuid ? '✅' : '❌'} User Account UUID: ${metadata.userAccountUuid || 'missing'}`) - console.log(` ${metadata.terminalType ? '✅' : '❌'} Terminal Type: ${metadata.terminalType || 'missing'}`) + console.log( + ` ${metadata.organizationId ? '✅' : '❌'} Organization ID: ${metadata.organizationId || 'missing'}`, + ) + console.log( + ` ${metadata.userAccountUuid ? '✅' : '❌'} User Account UUID: ${metadata.userAccountUuid || 'missing'}`, + ) + console.log( + ` ${metadata.terminalType ? '✅' : '❌'} Terminal Type: ${metadata.terminalType || 'missing'}`, + ) // Check observations const observations = traceDetails.observations || [] - const generations = observations.filter(o => o.type === 'GENERATION') - const events = observations.filter(o => o.type === 'EVENT') - const spans = observations.filter(o => o.type === 'SPAN') + const generations = observations.filter((o) => o.type === 'GENERATION') + const events = observations.filter((o) => o.type === 'EVENT') + const spans = observations.filter((o) => o.type === 'SPAN') validationResults.observations.generations = generations.length validationResults.observations.events = events.length validationResults.observations.spans = spans.length console.log('\n 📊 Observations:') - console.log(` ${generations.length > 0 ? '✅' : '❌'} Generations: ${generations.length}`) + console.log( + ` ${generations.length > 0 ? '✅' : '❌'} Generations: ${generations.length}`, + ) console.log(` ${events.length > 0 ? '✅' : '❌'} Events: ${events.length}`) console.log(` ${spans.length > 0 ? '✅' : '❌'} Spans: ${spans.length}`) @@ -149,18 +157,22 @@ async function validateComprehensive() { console.log('\n 🤖 Generation details:') console.log(` Model: ${gen.model}`) - console.log(` ${gen.usage?.total ? '✅' : '❌'} Token usage: ${gen.usage?.input || 0} in, ${gen.usage?.output || 0} out`) - console.log(` ${gen.metadata?.cache ? '✅' : '❌'} Cache tokens: ${gen.metadata?.cache?.read || 0} read, ${gen.metadata?.cache?.creation || 0} creation`) + console.log( + ` ${gen.usage?.total ? '✅' : '❌'} Token usage: ${gen.usage?.input || 0} in, ${gen.usage?.output || 0} out`, + ) + console.log( + ` ${gen.metadata?.cache ? '✅' : '❌'} Cache tokens: ${gen.metadata?.cache?.read || 0} read, ${gen.metadata?.cache?.creation || 0} creation`, + ) console.log(` ${gen.metadata?.cost ? '✅' : '❌'} Cost: $${gen.metadata?.cost || 0}`) } // Check events if (events.length > 0) { - const eventTypes = [...new Set(events.map(e => e.name))] + const eventTypes = [...new Set(events.map((e) => e.name))] console.log(`\n 📌 Event types: ${eventTypes.join(', ')}`) - validationResults.metrics.tools = eventTypes.some(e => e.includes('tool')) - validationResults.metrics.code = eventTypes.some(e => e.includes('code')) + validationResults.metrics.tools = eventTypes.some((e) => e.includes('tool')) + validationResults.metrics.code = eventTypes.some((e) => e.includes('code')) } } @@ -171,11 +183,11 @@ async function validateComprehensive() { // Check scores const scores = summaryDetails.scores || [] - validationResults.scores.quality = scores.some(s => s.name === 'quality') - validationResults.scores.efficiency = scores.some(s => s.name === 'efficiency') + validationResults.scores.quality = scores.some((s) => s.name === 'quality') + validationResults.scores.efficiency = scores.some((s) => s.name === 'efficiency') console.log(` ${scores.length > 0 ? '✅' : '❌'} Scores: ${scores.length} found`) - scores.forEach(score => { + scores.forEach((score) => { console.log(` - ${score.name}: ${score.value} - ${score.comment || 'No comment'}`) }) @@ -189,7 +201,9 @@ async function validateComprehensive() { console.log(` Total tokens: ${output.totalTokens || 0}`) if (output.cacheTokens) { - console.log(` Cache tokens: ${output.cacheTokens.read || 0} read, ${output.cacheTokens.creation || 0} creation`) + console.log( + ` Cache tokens: ${output.cacheTokens.read || 0} read, ${output.cacheTokens.creation || 0} creation`, + ) } if (output.additionalMetrics) { @@ -197,7 +211,9 @@ async function validateComprehensive() { console.log(` Active time: ${output.additionalMetrics.activeTime || 0}s`) console.log(` Commits: ${output.additionalMetrics.commitCount || 0}`) console.log(` Pull requests: ${output.additionalMetrics.pullRequestCount || 0}`) - console.log(` Tool decisions: ${output.additionalMetrics.toolDecisions?.length || 0}`) + console.log( + ` Tool decisions: ${output.additionalMetrics.toolDecisions?.length || 0}`, + ) } } } @@ -223,14 +239,14 @@ async function validateComprehensive() { { name: 'Efficiency score', passed: validationResults.scores.efficiency }, ] - const passed = checks.filter(c => c.passed).length + const passed = checks.filter((c) => c.passed).length const total = checks.length - checks.forEach(check => { + checks.forEach((check) => { console.log(` ${check.passed ? '✅' : '❌'} ${check.name}`) }) - console.log(`\n Score: ${passed}/${total} (${Math.round(passed / total * 100)}%)`) + console.log(`\n Score: ${passed}/${total} (${Math.round((passed / total) * 100)}%)`) if (passed === total) { console.log('\n 🎉 ALL VALIDATIONS PASSED!') diff --git a/test/integration/e2e.integration.test.js b/test/integration/e2e.integration.test.js index 93b47e8..d035ec5 100644 --- a/test/integration/e2e.integration.test.js +++ b/test/integration/e2e.integration.test.js @@ -62,14 +62,14 @@ describe('End-to-End Integration', () => { let stdout = '' let stderr = '' - child.stdout.on('data', data => { + child.stdout.on('data', (data) => { stdout += data }) - child.stderr.on('data', data => { + child.stderr.on('data', (data) => { stderr += data }) - child.on('close', code => { + child.on('close', (code) => { if (code === 0) { resolve({ stdout, stderr }) } else { @@ -127,12 +127,15 @@ describe('End-to-End Integration', () => { expect(output).toContain('4') // Wait for telemetry to be processed - await new Promise(resolve => setTimeout(resolve, 3000)) + await new Promise((resolve) => setTimeout(resolve, 3000)) // Verify conversation trace was created - const trace = await langfuseClient.waitForTrace({ - name: 'conversation-', - }, { timeout: 10000 }) + const trace = await langfuseClient.waitForTrace( + { + name: 'conversation-', + }, + { timeout: 10000 }, + ) expect(trace).toBeDefined() expect(trace.input).toMatchObject({ @@ -144,11 +147,13 @@ describe('End-to-End Integration', () => { const fullTrace = await langfuseClient.getTrace(trace.id) // Should have at least one generation (possibly two with routing) - const generations = fullTrace.observations?.filter(o => o.type === 'GENERATION') || [] + const generations = fullTrace.observations?.filter((o) => o.type === 'GENERATION') || [] expect(generations.length).toBeGreaterThanOrEqual(1) // Verify model and token usage - const mainGeneration = generations.find(g => g.model?.includes('opus') || g.model?.includes('sonnet')) + const mainGeneration = generations.find( + (g) => g.model?.includes('opus') || g.model?.includes('sonnet'), + ) expect(mainGeneration).toBeDefined() expect(mainGeneration.usage?.total).toBeGreaterThan(0) }) @@ -169,13 +174,12 @@ describe('End-to-End Integration', () => { }) // Wait for processing - await new Promise(resolve => setTimeout(resolve, 5000)) + await new Promise((resolve) => setTimeout(resolve, 5000)) // Find the conversation trace const traces = await langfuseClient.getTraces(20) - const trace = traces.find(t => - t.name?.startsWith('conversation-') && - t.input?.prompt?.includes('hello.py'), + const trace = traces.find( + (t) => t.name?.startsWith('conversation-') && t.input?.prompt?.includes('hello.py'), ) expect(trace).toBeDefined() @@ -184,13 +188,13 @@ describe('End-to-End Integration', () => { const fullTrace = await langfuseClient.getTrace(trace.id) // Should have tool usage events (Write tool) - const events = fullTrace.observations?.filter(o => o.type === 'EVENT') || [] - const toolEvents = events.filter(e => e.name?.includes('tool-')) + const events = fullTrace.observations?.filter((o) => o.type === 'EVENT') || [] + const toolEvents = events.filter((e) => e.name?.includes('tool-')) expect(toolEvents.length).toBeGreaterThan(0) // Should have a Write tool event - const writeEvent = toolEvents.find(e => e.name?.includes('Write')) + const writeEvent = toolEvents.find((e) => e.name?.includes('Write')) expect(writeEvent).toBeDefined() // Verify file was created @@ -210,11 +214,11 @@ describe('End-to-End Integration', () => { await runClaude(prompt) // Wait for session to complete and summary to be created - await new Promise(resolve => setTimeout(resolve, 8000)) + await new Promise((resolve) => setTimeout(resolve, 8000)) // Look for session summary trace const traces = await langfuseClient.getTraces(20) - const summaryTrace = traces.find(t => t.name === 'session-summary') + const summaryTrace = traces.find((t) => t.name === 'session-summary') // Note: Summary trace is created on session finalization // In real usage, this happens when the server shuts down or after timeout @@ -230,7 +234,7 @@ describe('End-to-End Integration', () => { const scores = fullTrace.scores || [] expect(scores.length).toBeGreaterThan(0) - const qualityScore = scores.find(s => s.name === 'quality') + const qualityScore = scores.find((s) => s.name === 'quality') expect(qualityScore).toBeDefined() } }) @@ -242,13 +246,12 @@ describe('End-to-End Integration', () => { await runClaude('What is the meaning of life?') // Wait for processing - await new Promise(resolve => setTimeout(resolve, 5000)) + await new Promise((resolve) => setTimeout(resolve, 5000)) // Get recent traces const traces = await langfuseClient.getTraces(10) - const conversationTrace = traces.find(t => - t.name?.startsWith('conversation-') && - t.input?.prompt?.includes('meaning of life'), + const conversationTrace = traces.find( + (t) => t.name?.startsWith('conversation-') && t.input?.prompt?.includes('meaning of life'), ) expect(conversationTrace).toBeDefined() @@ -264,8 +267,8 @@ describe('End-to-End Integration', () => { const fullTrace = await langfuseClient.getTrace(conversationTrace.id) // Check for cost tracking in generation metadata - const generation = fullTrace.observations?.find(o => - o.type === 'GENERATION' && o.metadata?.cost, + const generation = fullTrace.observations?.find( + (o) => o.type === 'GENERATION' && o.metadata?.cost, ) if (generation) { diff --git a/test/integration/langfuse.integration.test.js b/test/integration/langfuse.integration.test.js index 634af80..6460f9b 100644 --- a/test/integration/langfuse.integration.test.js +++ b/test/integration/langfuse.integration.test.js @@ -126,17 +126,17 @@ describe('Langfuse Integration', () => { }) // Wait a bit for processing - await new Promise(resolve => setTimeout(resolve, 1000)) + await new Promise((resolve) => setTimeout(resolve, 1000)) // Get trace with observations const traces = await langfuseClient.getTraces(10, testSessionId) - const conversationTrace = traces.find(t => t.name?.startsWith('conversation-')) + const conversationTrace = traces.find((t) => t.name?.startsWith('conversation-')) expect(conversationTrace).toBeDefined() const fullTrace = await langfuseClient.getTrace(conversationTrace.id) // Verify generation observation - const generations = fullTrace.observations?.filter(o => o.type === 'GENERATION') || [] + const generations = fullTrace.observations?.filter((o) => o.type === 'GENERATION') || [] expect(generations).toHaveLength(1) const generation = generations[0] @@ -185,7 +185,7 @@ describe('Langfuse Integration', () => { }) // Wait for processing - await new Promise(resolve => setTimeout(resolve, 1000)) + await new Promise((resolve) => setTimeout(resolve, 1000)) // Verify session has the metrics data const traces = await langfuseClient.getTraces(10, testSessionId) @@ -221,14 +221,14 @@ describe('Langfuse Integration', () => { }) // Wait for processing - await new Promise(resolve => setTimeout(resolve, 1000)) + await new Promise((resolve) => setTimeout(resolve, 1000)) // Get updated trace const fullTrace = await langfuseClient.getTrace(conversationTrace.id) // Verify tool event - const events = fullTrace.observations?.filter(o => o.type === 'EVENT') || [] - const toolEvent = events.find(e => e.name?.includes('tool-')) + const events = fullTrace.observations?.filter((o) => o.type === 'EVENT') || [] + const toolEvent = events.find((e) => e.name?.includes('tool-')) expect(toolEvent).toBeDefined() expect(toolEvent.name).toBe('tool-Read') @@ -280,7 +280,7 @@ describe('Langfuse Integration', () => { }) // Wait for processing - await new Promise(resolve => setTimeout(resolve, 2000)) + await new Promise((resolve) => setTimeout(resolve, 2000)) // Validate the complete flow const validation = await langfuseClient.verifyTelemetryFlow(testSessionId) diff --git a/test/metrics.integration.test.js b/test/metrics.integration.test.js index c92452f..3d82f14 100644 --- a/test/metrics.integration.test.js +++ b/test/metrics.integration.test.js @@ -17,32 +17,40 @@ describe('Metrics Integration Tests', () => { describe('Cost Metrics', () => { test('processes cost usage metrics with session', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, - { key: 'service.version', value: { stringValue: '3.0.0' } }, + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'claude-code' } }, + { key: 'service.version', value: { stringValue: '3.0.0' } }, + ], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.cost.usage', + description: 'Cost in USD', + unit: 'USD', + sum: { + dataPoints: [ + { + asDouble: 0.265, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-cost' } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.cost.usage', - description: 'Cost in USD', - unit: 'USD', - sum: { - dataPoints: [{ - asDouble: 0.265, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-cost' } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -60,32 +68,38 @@ describe('Metrics Integration Tests', () => { describe('Token Metrics', () => { test('processes token usage metrics', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.token.usage', + description: 'Token count', + unit: 'tokens', + sum: { + dataPoints: [ + { + asDouble: 1500, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-tokens' } }, + { key: 'type', value: { stringValue: 'input' } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.token.usage', - description: 'Token count', - unit: 'tokens', - sum: { - dataPoints: [{ - asDouble: 1500, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-tokens' } }, - { key: 'type', value: { stringValue: 'input' } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -101,31 +115,37 @@ describe('Metrics Integration Tests', () => { describe('Code Modification Metrics', () => { test('processes lines of code added metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.lines_of_code.count', + description: 'Lines of code', + unit: 'lines', + sum: { + dataPoints: [ + { + asDouble: 42, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-lines' } }, + { key: 'type', value: { stringValue: 'added' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.lines_of_code.count', - description: 'Lines of code', - unit: 'lines', - sum: { - dataPoints: [{ - asDouble: 42, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-lines' } }, - { key: 'type', value: { stringValue: 'added' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -139,31 +159,37 @@ describe('Metrics Integration Tests', () => { test('processes lines of code removed metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.lines_of_code.count', + description: 'Lines of code', + unit: 'lines', + sum: { + dataPoints: [ + { + asDouble: 15, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-lines' } }, + { key: 'type', value: { stringValue: 'removed' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.lines_of_code.count', - description: 'Lines of code', - unit: 'lines', - sum: { - dataPoints: [{ - asDouble: 15, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-lines' } }, - { key: 'type', value: { stringValue: 'removed' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -179,30 +205,36 @@ describe('Metrics Integration Tests', () => { describe('Git Metrics', () => { test('processes commit count metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.commit.count', + description: 'Git commits', + unit: 'commits', + sum: { + dataPoints: [ + { + asInt: '1', + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-git' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.commit.count', - description: 'Git commits', - unit: 'commits', - sum: { - dataPoints: [{ - asInt: '1', - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-git' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -218,33 +250,39 @@ describe('Metrics Integration Tests', () => { describe('Tool Decision Metrics', () => { test('processes code edit tool decision metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.code_edit_tool.decision', + description: 'Tool permission decisions', + unit: 'decisions', + gauge: { + dataPoints: [ + { + asInt: '1', + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-tool' } }, + { key: 'decision', value: { stringValue: 'accept' } }, + { key: 'tool', value: { stringValue: 'Write' } }, + { key: 'language', value: { stringValue: 'javascript' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.code_edit_tool.decision', - description: 'Tool permission decisions', - unit: 'decisions', - gauge: { - dataPoints: [{ - asInt: '1', - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-tool' } }, - { key: 'decision', value: { stringValue: 'accept' } }, - { key: 'tool', value: { stringValue: 'Write' } }, - { key: 'language', value: { stringValue: 'javascript' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -260,30 +298,36 @@ describe('Metrics Integration Tests', () => { describe('Session Metrics', () => { test('processes session count metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.session.count', + description: 'Session starts', + unit: 'sessions', + sum: { + dataPoints: [ + { + asInt: '1', + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-start' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.session.count', - description: 'Session starts', - unit: 'sessions', - sum: { - dataPoints: [{ - asInt: '1', - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-start' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -299,30 +343,36 @@ describe('Metrics Integration Tests', () => { describe('Active Time Metrics', () => { test('processes active time total metric', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.active_time.total', + description: 'Active usage time', + unit: 'seconds', + sum: { + dataPoints: [ + { + asDouble: 300.5, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session-time' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.active_time.total', - description: 'Active usage time', - unit: 'seconds', - sum: { - dataPoints: [{ - asDouble: 300.5, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session-time' } }, - ], - }], - }, - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -338,24 +388,32 @@ describe('Metrics Integration Tests', () => { describe('Error Handling', () => { test('handles metrics without session ID gracefully', async () => { const metric = { - resourceMetrics: [{ - resource: { - attributes: [], - }, - scopeMetrics: [{ - scope: { name: 'claude-code-telemetry' }, - metrics: [{ - name: 'claude_code.cost.usage', - sum: { - dataPoints: [{ - asDouble: 0.1, - timeUnixNano: Date.now() * 1000000, - attributes: [], // No session ID - }], + resourceMetrics: [ + { + resource: { + attributes: [], + }, + scopeMetrics: [ + { + scope: { name: 'claude-code-telemetry' }, + metrics: [ + { + name: 'claude_code.cost.usage', + sum: { + dataPoints: [ + { + asDouble: 0.1, + timeUnixNano: Date.now() * 1000000, + attributes: [], // No session ID + }, + ], + }, + }, + ], }, - }], - }], - }], + ], + }, + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { diff --git a/test/server.test.js b/test/server.test.js index 1dab36a..63cca7c 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -53,23 +53,25 @@ describeIntegration('OTLP Server Integration Tests', () => { describe('OTLP Endpoints', () => { test('POST /v1/logs accepts valid OTLP logs', async () => { const testLog = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'test-service' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'test-service' } }], + }, + scopeLogs: [ + { + scope: { name: 'test-scope' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'test log message' }, + attributes: [{ key: 'test.attribute', value: { stringValue: 'test-value' } }], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'test-scope' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'test log message' }, - attributes: [ - { key: 'test.attribute', value: { stringValue: 'test-value' } }, - ], - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -85,17 +87,19 @@ describeIntegration('OTLP Server Integration Tests', () => { test('POST /v1/metrics accepts valid OTLP metrics', async () => { const testMetric = { - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'test-service' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'test-service' } }], + }, + scopeMetrics: [ + { + scope: { name: 'test-scope' }, + metrics: [], + }, ], }, - scopeMetrics: [{ - scope: { name: 'test-scope' }, - metrics: [], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/metrics`, { @@ -111,17 +115,19 @@ describeIntegration('OTLP Server Integration Tests', () => { test('POST /v1/traces accepts valid OTLP traces', async () => { const testTrace = { - resourceSpans: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'test-service' } }, + resourceSpans: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'test-service' } }], + }, + scopeSpans: [ + { + scope: { name: 'test-scope' }, + spans: [], + }, ], }, - scopeSpans: [{ - scope: { name: 'test-scope' }, - spans: [], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/traces`, { @@ -140,27 +146,33 @@ describeIntegration('OTLP Server Integration Tests', () => { test('Processes user prompt event correctly', async () => { const sessionId = `test-session-${Date.now()}` const userPromptLog = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, - { key: 'service.version', value: { stringValue: '1.0.0' } }, - ], - }, - scopeLogs: [{ - scope: { name: 'com.anthropic.claude_code.events' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.user_prompt' }, + resourceLogs: [ + { + resource: { attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'user.email', value: { stringValue: 'test@example.com' } }, - { key: 'prompt', value: { stringValue: 'Test prompt' } }, - { key: 'prompt_length', value: { stringValue: '11' } }, + { key: 'service.name', value: { stringValue: 'claude-code' } }, + { key: 'service.version', value: { stringValue: '1.0.0' } }, ], - }], - }], - }], + }, + scopeLogs: [ + { + scope: { name: 'com.anthropic.claude_code.events' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.user_prompt' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'user.email', value: { stringValue: 'test@example.com' } }, + { key: 'prompt', value: { stringValue: 'Test prompt' } }, + { key: 'prompt_length', value: { stringValue: '11' } }, + ], + }, + ], + }, + ], + }, + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -179,28 +191,32 @@ describeIntegration('OTLP Server Integration Tests', () => { test('Processes API request event correctly', async () => { const sessionId = `test-session-${Date.now()}` const apiRequestLog = { - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'com.anthropic.claude_code.events' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.api_request' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'model', value: { stringValue: 'claude-3-5-sonnet-20241022' } }, + { key: 'input_tokens', value: { stringValue: '100' } }, + { key: 'output_tokens', value: { stringValue: '200' } }, + { key: 'cost_usd', value: { stringValue: '0.0015' } }, + { key: 'duration_ms', value: { stringValue: '1500' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'com.anthropic.claude_code.events' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.api_request' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'model', value: { stringValue: 'claude-3-5-sonnet-20241022' } }, - { key: 'input_tokens', value: { stringValue: '100' } }, - { key: 'output_tokens', value: { stringValue: '200' } }, - { key: 'cost_usd', value: { stringValue: '0.0015' } }, - { key: 'duration_ms', value: { stringValue: '1500' } }, - ], - }], - }], - }], + ], } const response = await fetch(`${baseUrl}/v1/logs`, { @@ -254,24 +270,28 @@ describeIntegration('OTLP Server Integration Tests', () => { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'com.anthropic.claude_code.events' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.user_prompt' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'prompt', value: { stringValue: 'Integration test prompt' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'com.anthropic.claude_code.events' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.user_prompt' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'prompt', value: { stringValue: 'Integration test prompt' } }, - ], - }], - }], - }], + ], }), }) @@ -280,28 +300,32 @@ describeIntegration('OTLP Server Integration Tests', () => { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'com.anthropic.claude_code.events' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000 + 1000000, + body: { stringValue: 'claude_code.api_request' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'model', value: { stringValue: 'claude-3-5-sonnet-20241022' } }, + { key: 'input_tokens', value: { stringValue: '150' } }, + { key: 'output_tokens', value: { stringValue: '300' } }, + { key: 'cost_usd', value: { stringValue: '0.00225' } }, + { key: 'duration_ms', value: { stringValue: '2000' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'com.anthropic.claude_code.events' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000 + 1000000, - body: { stringValue: 'claude_code.api_request' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'model', value: { stringValue: 'claude-3-5-sonnet-20241022' } }, - { key: 'input_tokens', value: { stringValue: '150' } }, - { key: 'output_tokens', value: { stringValue: '300' } }, - { key: 'cost_usd', value: { stringValue: '0.00225' } }, - { key: 'duration_ms', value: { stringValue: '2000' } }, - ], - }], - }], - }], + ], }), }) @@ -312,32 +336,36 @@ describeIntegration('OTLP Server Integration Tests', () => { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + scope: { name: 'com.anthropic.claude_code.events' }, + logRecords: [ + { + timeUnixNano: Date.now() * 1000000 + 2000000, + body: { stringValue: 'claude_code.tool_result' }, + attributes: [ + { key: 'session.id', value: { stringValue: sessionId } }, + { key: 'tool_name', value: { stringValue: 'Write' } }, + { key: 'success', value: { stringValue: 'true' } }, + { key: 'duration_ms', value: { stringValue: '50' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - scope: { name: 'com.anthropic.claude_code.events' }, - logRecords: [{ - timeUnixNano: Date.now() * 1000000 + 2000000, - body: { stringValue: 'claude_code.tool_result' }, - attributes: [ - { key: 'session.id', value: { stringValue: sessionId } }, - { key: 'tool_name', value: { stringValue: 'Write' } }, - { key: 'success', value: { stringValue: 'true' } }, - { key: 'duration_ms', value: { stringValue: '50' } }, - ], - }], - }], - }], + ], }), }) break // Success, exit retry loop } catch (error) { if (retry === 2) throw error // Last retry, re-throw - await new Promise(resolve => setTimeout(resolve, 100)) // Wait before retry + await new Promise((resolve) => setTimeout(resolve, 100)) // Wait before retry } } diff --git a/test/setup.js b/test/setup.js index 98292f2..0c744e2 100644 --- a/test/setup.js +++ b/test/setup.js @@ -23,9 +23,10 @@ expect.extend({ const pass = Number.isInteger(received) && received >= 1 && received <= 65535 return { pass, - message: () => pass - ? `expected ${received} not to be a valid port` - : `expected ${received} to be a valid port (1-65535)`, + message: () => + pass + ? `expected ${received} not to be a valid port` + : `expected ${received} to be a valid port (1-65535)`, } }, }) @@ -43,5 +44,5 @@ if (process.env.DEBUG_TESTS !== 'true') { } // Add skipIf helper for conditional test execution -global.describe.skipIf = (condition) => condition ? describe.skip : describe -global.test.skipIf = (condition) => condition ? test.skip : test +global.describe.skipIf = (condition) => (condition ? describe.skip : describe) +global.test.skipIf = (condition) => (condition ? test.skip : test) diff --git a/test/testServer.js b/test/testServer.js index a69bc70..87e10b1 100644 --- a/test/testServer.js +++ b/test/testServer.js @@ -67,7 +67,7 @@ async function startTestServer(testFile, customEnv = {}) { // Server not ready yet } - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) attempts++ } @@ -95,7 +95,7 @@ async function stopTestServer(serverProcess) { } // Wait for process to exit - await new Promise(resolve => { + await new Promise((resolve) => { if (serverProcess.exitCode !== null) { resolve() return @@ -114,7 +114,7 @@ async function stopTestServer(serverProcess) { }) // Small delay to ensure port is released - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) } /** diff --git a/test/unit/eventProcessor.test.js b/test/unit/eventProcessor.test.js index c4e5399..1535465 100644 --- a/test/unit/eventProcessor.test.js +++ b/test/unit/eventProcessor.test.js @@ -21,12 +21,13 @@ jest.mock('../../src/sessionHandler', () => ({ // Simple implementation for testing const result = {} if (attrs && Array.isArray(attrs)) { - attrs.forEach(attr => { + attrs.forEach((attr) => { const key = attr.key - const value = attr.value?.stringValue || - attr.value?.intValue || - attr.value?.doubleValue || - attr.value?.boolValue + const value = + attr.value?.stringValue || + attr.value?.intValue || + attr.value?.doubleValue || + attr.value?.boolValue if (key && value !== undefined) { result[key] = value } @@ -224,9 +225,7 @@ describe('Event Processor', () => { const logRecord = { body: { stringValue: 'claude_code.user_prompt' }, timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'prompt_length', value: { stringValue: '10' } }, - ], + attributes: [{ key: 'prompt_length', value: { stringValue: '10' } }], } const resource = {} diff --git a/test/unit/offlineFallback.test.js b/test/unit/offlineFallback.test.js new file mode 100644 index 0000000..6507200 --- /dev/null +++ b/test/unit/offlineFallback.test.js @@ -0,0 +1,119 @@ +const fs = require('fs') +const path = require('path') +const os = require('os') + +const { writeReference, getFallbackPath } = require('../../src/offlineFallback') + +describe('offlineFallback', () => { + let tmpDir + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fallback-test-')) + process.env.FALLBACK_FILE = path.join(tmpDir, 'test-fallback.jsonl') + }) + + afterEach(() => { + delete process.env.FALLBACK_FILE + fs.rmSync(tmpDir, { recursive: true, force: true }) + }) + + describe('getFallbackPath', () => { + test('returns FALLBACK_FILE env var when set', () => { + process.env.FALLBACK_FILE = '/custom/path.jsonl' + expect(getFallbackPath()).toBe('/custom/path.jsonl') + }) + + test('returns default path when env var is not set', () => { + delete process.env.FALLBACK_FILE + const expected = path.join(process.env.HOME || '/tmp', '.claude', 'telemetry-fallback.jsonl') + expect(getFallbackPath()).toBe(expected) + }) + }) + + describe('writeReference', () => { + test('writes a session reference as JSONL', () => { + const session = { + sessionId: 'sess-123', + metadata: { userId: 'user-1' }, + organizationId: 'org-1', + userAccountUuid: 'uuid-1', + createdAt: new Date('2025-01-01T00:00:00Z'), + conversationCount: 5, + apiCallCount: 10, + totalTokens: 5000, + totalCost: 1.23, + } + + const ref = writeReference(session) + + expect(ref.sessionId).toBe('sess-123') + expect(ref.userId).toBe('user-1') + expect(ref.organizationId).toBe('org-1') + expect(ref.userAccountUuid).toBe('uuid-1') + expect(ref.createdAt).toBe('2025-01-01T00:00:00.000Z') + expect(ref.failedAt).toBeDefined() + expect(ref.stats.conversationCount).toBe(5) + expect(ref.stats.apiCallCount).toBe(10) + expect(ref.stats.totalTokens).toBe(5000) + expect(ref.stats.totalCost).toBe(1.23) + + const content = fs.readFileSync(getFallbackPath(), 'utf8') + const parsed = JSON.parse(content.trim()) + expect(parsed.sessionId).toBe('sess-123') + }) + + test('creates directory if it does not exist', () => { + const nestedPath = path.join(tmpDir, 'nested', 'dir', 'fallback.jsonl') + process.env.FALLBACK_FILE = nestedPath + + writeReference({ + sessionId: 'sess-1', + conversationCount: 0, + apiCallCount: 0, + totalTokens: 0, + totalCost: 0, + }) + + expect(fs.existsSync(nestedPath)).toBe(true) + }) + + test('appends multiple references as separate lines', () => { + const session1 = { + sessionId: 'sess-1', + conversationCount: 1, + apiCallCount: 1, + totalTokens: 100, + totalCost: 0.1, + } + const session2 = { + sessionId: 'sess-2', + conversationCount: 2, + apiCallCount: 2, + totalTokens: 200, + totalCost: 0.2, + } + + writeReference(session1) + writeReference(session2) + + const lines = fs.readFileSync(getFallbackPath(), 'utf8').trim().split('\n') + expect(lines).toHaveLength(2) + expect(JSON.parse(lines[0]).sessionId).toBe('sess-1') + expect(JSON.parse(lines[1]).sessionId).toBe('sess-2') + }) + + test('handles missing metadata gracefully', () => { + const session = { + sessionId: 'sess-no-meta', + conversationCount: 0, + apiCallCount: 0, + totalTokens: 0, + totalCost: 0, + } + + const ref = writeReference(session) + expect(ref.userId).toBeUndefined() + expect(ref.createdAt).toBeUndefined() + }) + }) +}) diff --git a/test/unit/requestHandlers.test.js b/test/unit/requestHandlers.test.js index edccfe6..0f2134f 100644 --- a/test/unit/requestHandlers.test.js +++ b/test/unit/requestHandlers.test.js @@ -10,7 +10,7 @@ jest.mock('../../src/sessionHandler', () => ({ extractAttributesArray: jest.fn((attrs) => { const result = {} if (attrs) { - attrs.forEach(attr => { + attrs.forEach((attr) => { result[attr.key] = attr.value.stringValue || attr.value.intValue || attr.value.doubleValue }) } @@ -53,12 +53,14 @@ describe('Request Handlers', () => { describe('handleTraces', () => { test('accepts valid traces payload', () => { const data = JSON.stringify({ - resourceSpans: [{ - resource: { - attributes: [], + resourceSpans: [ + { + resource: { + attributes: [], + }, + scopeSpans: [], }, - scopeSpans: [], - }], + ], }) handleTraces(data, mockRes, mockSessions, mockLangfuse) @@ -85,28 +87,34 @@ describe('Request Handlers', () => { const { SessionHandler } = require('../../src/sessionHandler') const data = JSON.stringify({ - resourceMetrics: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'claude_code.cost.usage', + sum: { + dataPoints: [ + { + asDouble: 0.15, + timeUnixNano: Date.now() * 1000000, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session' } }, + { key: 'model', value: { stringValue: 'claude-3-opus' } }, + ], + }, + ], + }, + }, + ], + }, ], }, - scopeMetrics: [{ - metrics: [{ - name: 'claude_code.cost.usage', - sum: { - dataPoints: [{ - asDouble: 0.15, - timeUnixNano: Date.now() * 1000000, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session' } }, - { key: 'model', value: { stringValue: 'claude-3-opus' } }, - ], - }], - }, - }], - }], - }], + ], }) handleMetrics(data, mockRes, mockSessions, mockLangfuse) @@ -121,20 +129,28 @@ describe('Request Handlers', () => { test('handles metrics without session ID', () => { const data = JSON.stringify({ - resourceMetrics: [{ - resource: { attributes: [] }, - scopeMetrics: [{ - metrics: [{ - name: 'claude_code.cost.usage', - sum: { - dataPoints: [{ - asDouble: 0.1, - attributes: [], - }], + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'claude_code.cost.usage', + sum: { + dataPoints: [ + { + asDouble: 0.1, + attributes: [], + }, + ], + }, + }, + ], }, - }], - }], - }], + ], + }, + ], }) handleMetrics(data, mockRes, mockSessions, mockLangfuse) @@ -148,20 +164,28 @@ describe('Request Handlers', () => { process.env.LOG_LEVEL = 'debug' const data = JSON.stringify({ - resourceMetrics: [{ - resource: { attributes: [] }, - scopeMetrics: [{ - metrics: [{ - name: 'test.metric', - gauge: { - dataPoints: [{ - asDouble: 42, - attributes: [], - }], + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'test.metric', + gauge: { + dataPoints: [ + { + asDouble: 42, + attributes: [], + }, + ], + }, + }, + ], }, - }], - }], - }], + ], + }, + ], }) handleMetrics(data, mockRes, mockSessions, mockLangfuse) @@ -186,23 +210,27 @@ describe('Request Handlers', () => { describe('handleLogs', () => { test('processes logs with session ID', () => { const data = JSON.stringify({ - resourceLogs: [{ - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'claude-code' } }, + resourceLogs: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'claude-code' } }], + }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.user_prompt' }, + attributes: [ + { key: 'session.id', value: { stringValue: 'test-session' } }, + { key: 'prompt', value: { stringValue: 'Test prompt' } }, + ], + }, + ], + }, ], }, - scopeLogs: [{ - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.user_prompt' }, - attributes: [ - { key: 'session.id', value: { stringValue: 'test-session' } }, - { key: 'prompt', value: { stringValue: 'Test prompt' } }, - ], - }], - }], - }], + ], }) handleLogs(data, mockRes, mockSessions, mockLangfuse) @@ -215,19 +243,25 @@ describe('Request Handlers', () => { test('creates session from user ID and timestamp', () => { const data = JSON.stringify({ - resourceLogs: [{ - resource: { attributes: [] }, - scopeLogs: [{ - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'claude_code.user_prompt' }, - attributes: [ - { key: 'user.email', value: { stringValue: 'test@example.com' } }, - { key: 'event.timestamp', value: { stringValue: '2024-01-01T12:00:00Z' } }, - ], - }], - }], - }], + resourceLogs: [ + { + resource: { attributes: [] }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'claude_code.user_prompt' }, + attributes: [ + { key: 'user.email', value: { stringValue: 'test@example.com' } }, + { key: 'event.timestamp', value: { stringValue: '2024-01-01T12:00:00Z' } }, + ], + }, + ], + }, + ], + }, + ], }) handleLogs(data, mockRes, mockSessions, mockLangfuse) @@ -240,16 +274,22 @@ describe('Request Handlers', () => { test('logs without session are ignored', () => { const data = JSON.stringify({ - resourceLogs: [{ - resource: { attributes: [] }, - scopeLogs: [{ - logRecords: [{ - timeUnixNano: Date.now() * 1000000, - body: { stringValue: 'test log' }, - attributes: [], - }], - }], - }], + resourceLogs: [ + { + resource: { attributes: [] }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: Date.now() * 1000000, + body: { stringValue: 'test log' }, + attributes: [], + }, + ], + }, + ], + }, + ], }) handleLogs(data, mockRes, mockSessions, mockLangfuse) diff --git a/test/unit/serverHelpers.test.js b/test/unit/serverHelpers.test.js index a1c06bb..4f538ac 100644 --- a/test/unit/serverHelpers.test.js +++ b/test/unit/serverHelpers.test.js @@ -133,9 +133,7 @@ describe('Server Helpers', () => { finalize: jest.fn().mockRejectedValue(new Error('Finalize failed')), } - const sessions = new Map([ - ['session-1', mockSession], - ]) + const sessions = new Map([['session-1', mockSession]]) const cleaned = await cleanupSessions(sessions, 300000) @@ -169,9 +167,7 @@ describe('Server Helpers', () => { finalize: jest.fn().mockRejectedValue(new Error('Finalize failed')), } - const sessions = new Map([ - ['session-1', mockSession], - ]) + const sessions = new Map([['session-1', mockSession]]) await finalizeAllSessions(sessions) @@ -251,8 +247,14 @@ describe('Server Helpers', () => { setCorsHeaders(res) expect(res.setHeader).toHaveBeenCalledWith('Access-Control-Allow-Origin', '*') - expect(res.setHeader).toHaveBeenCalledWith('Access-Control-Allow-Methods', 'POST, GET, OPTIONS') - expect(res.setHeader).toHaveBeenCalledWith('Access-Control-Allow-Headers', 'Content-Type, Authorization') + expect(res.setHeader).toHaveBeenCalledWith( + 'Access-Control-Allow-Methods', + 'POST, GET, OPTIONS', + ) + expect(res.setHeader).toHaveBeenCalledWith( + 'Access-Control-Allow-Headers', + 'Content-Type, Authorization', + ) }) }) diff --git a/test/unit/sessionHandler.test.js b/test/unit/sessionHandler.test.js index 28a5323..10cb7b4 100644 --- a/test/unit/sessionHandler.test.js +++ b/test/unit/sessionHandler.test.js @@ -60,10 +60,14 @@ describe('SessionHandler', () => { flushAsync: jest.fn(() => Promise.resolve()), } - session = new SessionHandler('test-session-id', { - 'service.name': 'test-service', - 'service.version': '1.0.0', - }, mockLangfuseInstance) + session = new SessionHandler( + 'test-session-id', + { + 'service.name': 'test-service', + 'service.version': '1.0.0', + }, + mockLangfuseInstance, + ) }) describe('constructor', () => { @@ -79,7 +83,9 @@ describe('SessionHandler', () => { }) test('throws error if sessionId is not provided', () => { - expect(() => new SessionHandler(null, {}, mockLangfuseInstance)).toThrow('SessionHandler requires a sessionId') + expect(() => new SessionHandler(null, {}, mockLangfuseInstance)).toThrow( + 'SessionHandler requires a sessionId', + ) }) }) @@ -520,18 +526,24 @@ describe('SessionHandler', () => { test('tracks consecutive tool calls in sequence', () => { // First tool in sequence - session.handleToolResult({ - tool_name: 'Read', - success: 'true', - duration_ms: '100', - }, '2024-07-31T10:00:00Z') + session.handleToolResult( + { + tool_name: 'Read', + success: 'true', + duration_ms: '100', + }, + '2024-07-31T10:00:00Z', + ) // Second tool in sequence - session.handleToolResult({ - tool_name: 'Edit', - success: 'true', - duration_ms: '200', - }, '2024-07-31T10:00:01Z') + session.handleToolResult( + { + tool_name: 'Edit', + success: 'true', + duration_ms: '200', + }, + '2024-07-31T10:00:01Z', + ) expect(session.toolSequence).toHaveLength(2) expect(session.toolSequence[0].name).toBe('Read') @@ -744,9 +756,7 @@ describe('SessionHandler', () => { describe('Helper Functions', () => { describe('extractAttributesArray', () => { test('extracts string attributes', () => { - const attributes = [ - { key: 'name', value: { stringValue: 'test' } }, - ] + const attributes = [{ key: 'name', value: { stringValue: 'test' } }] const result = extractAttributesArray(attributes) expect(result).toEqual({ name: 'test' }) }) @@ -771,10 +781,7 @@ describe('Helper Functions', () => { key: 'items', value: { arrayValue: { - values: [ - { stringValue: 'item1' }, - { stringValue: 'item2' }, - ], + values: [{ stringValue: 'item1' }, { stringValue: 'item2' }], }, }, }, @@ -819,9 +826,7 @@ describe('Helper Functions', () => { }) test('handles unknown value types', () => { - const attributes = [ - { key: 'unknown', value: { unknownType: 'value' } }, - ] + const attributes = [{ key: 'unknown', value: { unknownType: 'value' } }] const result = extractAttributesArray(attributes) expect(result).toEqual({ unknown: null })