From 052125cc8525c50da3a5f5712ca6155ca1e8a5aa Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Wed, 8 Apr 2026 21:41:44 +1000 Subject: [PATCH 1/4] feat(md): add --format flag for rev-iso/legacy filename formats --- src/cli.ts | 2 + src/md-export.ts | 164 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 131 insertions(+), 35 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 078493a..2711873 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1015,12 +1015,14 @@ export function buildCli() { .command('md') .description('Export bookmarks as individual markdown files') .option('--force', 'Re-export all bookmarks (overwrite existing files)') + .option('--format ', 'Filename format: rev-iso (default), legacy', (v: string) => v as 'legacy' | 'rev-iso', 'rev-iso') .action(safe(async (options) => { if (!requireIndex()) return; let lastLine = ''; const spinner = createSpinner(() => lastLine); const result = await exportBookmarks({ force: options.force, + filenameFormat: options.format, onProgress: (s) => { lastLine = s; spinner.update(); diff --git a/src/md-export.ts b/src/md-export.ts index 50897f5..36ff6f6 100644 --- a/src/md-export.ts +++ b/src/md-export.ts @@ -7,19 +7,26 @@ * full tweet text, and [[wikilinks]] to wiki category/domain/entity pages. * No LLM required — fast, deterministic, portable. * - * Output: ~/.ft-bookmarks/md/bookmarks/--.md + * Output: ~/.ft-bookmarks/md/bookmarks/__--.md */ -import fs from 'node:fs'; -import path from 'node:path'; -import { ensureDir, writeMd } from './fs.js'; -import { mdDir } from './paths.js'; -import { listBookmarks, countBookmarks, type BookmarkTimelineItem } from './bookmarks-db.js'; -import { slug } from './md.js'; +import fs from "node:fs"; +import path from "node:path"; +import { ensureDir, writeMd } from "./fs.js"; +import { mdDir } from "./paths.js"; +import { + listBookmarks, + countBookmarks, + type BookmarkTimelineItem, +} from "./bookmarks-db.js"; +import { slug } from "./md.js"; + +export type FilenameFormat = "legacy" | "rev-iso"; export interface ExportOptions { force?: boolean; onProgress?: (status: string) => void; + filenameFormat?: FilenameFormat; } export interface ExportResult { @@ -29,58 +36,136 @@ export interface ExportResult { elapsed: number; } +const DATE_STR_REGEX = RegExp( + /^(\w{3})\s+(\w{3})\s+(\d{1,2})\s+\d{2}:\d{2}:\d{2}\s+[+-]\d{4}\s+(\d{4})$/, +); + function bookmarksDir(): string { - return path.join(mdDir(), 'bookmarks'); + return path.join(mdDir(), "bookmarks"); } -function bookmarkFilename(b: BookmarkTimelineItem): string { - const date = (b.postedAt ?? b.bookmarkedAt ?? '').slice(0, 10) || 'undated'; - const author = b.authorHandle ? slug(b.authorHandle) : 'unknown'; +function parsePostedAt(dateStr: string | null | undefined): { + year: string; + month: string; + day: string; + dow: string; +} | null { + if (!dateStr) return null; + + if (dateStr.length === 10 && dateStr.includes("-")) { + const year = parseInt(dateStr.slice(0, 4)); + const month = parseInt(dateStr.slice(5, 7)) - 1; + const day = parseInt(dateStr.slice(8, 10)); + const dowNum = new Date(year, month, day).getDay(); + const dow = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][dowNum]; + return { + year: dateStr.slice(0, 4), + month: dateStr.slice(5, 7), + day: dateStr.slice(8, 10), + dow, + }; + } + + const match = dateStr.match(DATE_STR_REGEX); + if (match) { + const [, dow, , day, year] = match; + const monthNames = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ]; + const monthStr = match[2]; + const month = String(monthNames.indexOf(monthStr) + 1).padStart(2, "0"); + return { year, month, day: day.padStart(2, "0"), dow }; + } + + return null; +} + +type Author = string | null | undefined; + +function bookmarkFilenameLegacy(b: BookmarkTimelineItem, a: Author): string { + if (!a) a = "unknown"; + const date = (b.postedAt ?? b.bookmarkedAt ?? "").slice(0, 10) || "undated"; + const author = a; const textSlug = slug(b.text.slice(0, 50)) || b.id; return `${date}-${author}-${textSlug}.md`; } +function bookmarkFilename( + b: BookmarkTimelineItem, + format: FilenameFormat = "rev-iso", +): string { + const author = b.authorHandle ? slug(b.authorHandle) : "unknown"; + if (format === "legacy") { + return bookmarkFilenameLegacy(b, author); + } + + const dateInfo = parsePostedAt(b.postedAt ?? b.bookmarkedAt ?? null); + const textSlug = slug(b.text.slice(0, 50)) || b.id; + + if (dateInfo) { + return `${dateInfo.year}_${dateInfo.month}_${dateInfo.day}-${dateInfo.dow}-${author}-${textSlug}.md`; + } + + return `undated-${author}-${textSlug}.md`; +} + function buildBookmarkMd(b: BookmarkTimelineItem): string { const lines: string[] = []; // ── Frontmatter ───────────────────────────────────────────────────── - lines.push('---'); + lines.push("---"); if (b.authorHandle) lines.push(`author: "@${b.authorHandle}"`); - if (b.authorName) lines.push(`author_name: "${b.authorName.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, ' ')}"`); + if (b.authorName) + lines.push( + `author_name: "${b.authorName.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, " ")}"`, + ); if (b.postedAt) lines.push(`posted_at: ${b.postedAt.slice(0, 10)}`); - if (b.bookmarkedAt) lines.push(`bookmarked_at: ${b.bookmarkedAt.slice(0, 10)}`); + if (b.bookmarkedAt) + lines.push(`bookmarked_at: ${b.bookmarkedAt.slice(0, 10)}`); if (b.primaryCategory) lines.push(`category: ${b.primaryCategory}`); if (b.primaryDomain) lines.push(`domain: ${b.primaryDomain}`); - if (b.categories.length > 0) lines.push(`categories: [${b.categories.join(', ')}]`); - if (b.domains.length > 0) lines.push(`domains: [${b.domains.join(', ')}]`); + if (b.categories.length > 0) + lines.push(`categories: [${b.categories.join(", ")}]`); + if (b.domains.length > 0) lines.push(`domains: [${b.domains.join(", ")}]`); lines.push(`source_url: ${b.url}`); lines.push(`tweet_id: "${b.tweetId}"`); if (b.likeCount) lines.push(`likes: ${b.likeCount}`); if (b.repostCount) lines.push(`reposts: ${b.repostCount}`); if (b.viewCount) lines.push(`views: ${b.viewCount}`); - lines.push('---'); - lines.push(''); + lines.push("---"); + lines.push(""); // ── Title ─────────────────────────────────────────────────────────── - const author = b.authorHandle ? `@${b.authorHandle}` : 'Unknown'; + const author = b.authorHandle ? `@${b.authorHandle}` : "Unknown"; lines.push(`# ${author}`); - lines.push(''); + lines.push(""); // ── Body ──────────────────────────────────────────────────────────── lines.push(b.text); - lines.push(''); + lines.push(""); // ── Links ─────────────────────────────────────────────────────────── if (b.links.length > 0) { - lines.push('## Links'); + lines.push("## Links"); for (const link of b.links) lines.push(`- ${link}`); - lines.push(''); + lines.push(""); } if (b.githubUrls.length > 0) { - lines.push('## GitHub'); + lines.push("## GitHub"); for (const url of b.githubUrls) lines.push(`- ${url}`); - lines.push(''); + lines.push(""); } // ── Wikilinks to wiki pages ───────────────────────────────────────── @@ -90,20 +175,23 @@ function buildBookmarkMd(b: BookmarkTimelineItem): string { if (b.authorHandle) refs.push(`[[entities/${slug(b.authorHandle)}]]`); if (refs.length > 0) { - lines.push('## Related'); + lines.push("## Related"); for (const ref of refs) lines.push(`- ${ref}`); - lines.push(''); + lines.push(""); } // ── Source ────────────────────────────────────────────────────────── lines.push(`[Original tweet](${b.url})`); - lines.push(''); + lines.push(""); - return lines.join('\n'); + return lines.join("\n"); } -export async function exportBookmarks(options: ExportOptions = {}): Promise { - const progress = options.onProgress ?? ((s: string) => fs.writeSync(2, s + '\n')); +export async function exportBookmarks( + options: ExportOptions = {}, +): Promise { + const progress = + options.onProgress ?? ((s: string) => fs.writeSync(2, s + "\n")); const startTime = Date.now(); await ensureDir(bookmarksDir()); @@ -117,9 +205,11 @@ export async function exportBookmarks(options: ExportOptions = {}): Promise Date: Wed, 8 Apr 2026 21:41:44 +1000 Subject: [PATCH 2/4] feat(classify): add --fail-fast + colored err --- .gitignore | 1 + src/bookmark-classify-llm.ts | 25 ++++++++++++++++++------ src/bookmarks-viz.ts | 8 ++++---- src/cli.ts | 13 +++++++++---- src/engine.ts | 37 ++++++++++++++++++++++++++++++------ 5 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index e871be6..2e45f92 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ *.js.map .env .env.local +pnpm* diff --git a/src/bookmark-classify-llm.ts b/src/bookmark-classify-llm.ts index 82d7600..0aa394d 100644 --- a/src/bookmark-classify-llm.ts +++ b/src/bookmark-classify-llm.ts @@ -10,6 +10,7 @@ import { openDb, saveDb } from './db.js'; import { twitterBookmarksIndexPath } from './paths.js'; import type { ResolvedEngine } from './engine.js'; import { invokeEngine } from './engine.js'; +import { C, RESET } from './bookmarks-viz.js'; const BATCH_SIZE = 50; @@ -110,9 +111,9 @@ export interface LlmClassifyResult { } export async function classifyWithLlm( - options: { engine: ResolvedEngine; onBatch?: (done: number, total: number) => void }, + options: { engine: ResolvedEngine; onBatch?: (done: number, total: number) => void; failFast?: boolean }, ): Promise { - const { engine } = options; + const { engine, failFast = false } = options; const dbPath = twitterBookmarksIndexPath(); const db = await openDb(dbPath); @@ -170,7 +171,13 @@ export async function classifyWithLlm( saveDb(db, dbPath); } catch (err) { failed += batch.length; - process.stderr.write(` Batch ${batchCount} failed: ${(err as Error).message}\n`); + const errMsg = (err as Error).message; + const firstLine = errMsg.split('\n')[0]; + console.error(`\n${C.gold} Error: ${firstLine}${RESET}`); + if (failFast) { + console.error(`${C.gold} fail-fast enabled — stopping classification${RESET}`); + break; + } } } @@ -223,9 +230,9 @@ ${items}`; } export async function classifyDomainsWithLlm( - options: { engine: ResolvedEngine; all?: boolean; onBatch?: (done: number, total: number) => void }, + options: { engine: ResolvedEngine; all?: boolean; onBatch?: (done: number, total: number) => void; failFast?: boolean }, ): Promise { - const { engine } = options; + const { engine, failFast = false } = options; const dbPath = twitterBookmarksIndexPath(); const db = await openDb(dbPath); @@ -285,7 +292,13 @@ export async function classifyDomainsWithLlm( saveDb(db, dbPath); } catch (err) { failed += batch.length; - process.stderr.write(` Batch ${batchCount} failed: ${(err as Error).message}\n`); + const errMsg = (err as Error).message; + const firstLine = errMsg.split('\n')[0]; + console.error(`\n${C.gold} Error: ${firstLine}${RESET}`); + if (failFast) { + console.error(`${C.gold} fail-fast enabled — stopping classification${RESET}`); + break; + } } } diff --git a/src/bookmarks-viz.ts b/src/bookmarks-viz.ts index 78b6bb8..e326b9e 100644 --- a/src/bookmarks-viz.ts +++ b/src/bookmarks-viz.ts @@ -3,15 +3,15 @@ import { twitterBookmarksIndexPath } from './paths.js'; // ── ANSI helpers ───────────────────────────────────────────────────────────── -const ESC = '\x1b['; -const RESET = `${ESC}0m`; +export const ESC = '\x1b['; +export const RESET = `${ESC}0m`; const BOLD = `${ESC}1m`; const DIM = `${ESC}2m`; const rgb = (r: number, g: number, b: number) => `${ESC}38;2;${r};${g};${b}m`; // Palette — muted, tasteful -const C = { +export const C = { title: rgb(199, 146, 234), // soft lavender accent: rgb(130, 170, 255), // periwinkle warm: rgb(255, 180, 120), // peach @@ -93,7 +93,7 @@ function boxBottom(width: number): string { return C.dim + '╰' + '─'.repeat(width - 2) + '╯' + RESET; } function boxRow(content: string, width: number): string { - const stripped = content.replace(/\x1b\[[^m]*m/g, ''); + const stripped = content.replace(new RegExp(ESC + '\\[([^m]*m)', 'g'), ''); const pad = Math.max(0, width - 4 - stripped.length); return C.dim + '│ ' + RESET + content + ' '.repeat(pad) + C.dim + ' │' + RESET; } diff --git a/src/cli.ts b/src/cli.ts index 2711873..197060f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -767,12 +767,13 @@ export function buildCli() { .command('classify') .description('Classify bookmarks by category and domain using LLM (requires claude or codex CLI)') .option('--regex', 'Use simple regex classification instead of LLM') + .option('--fail-fast', 'Stop immediately on first classification failure') .action(safe(async (options) => { if (!requireData()) return; if (options.regex) { process.stderr.write('Classifying bookmarks (regex)...\n'); const result = await classifyAndRebuild(); - console.log(`Indexed ${result.recordCount} bookmarks \u2192 ${result.dbPath}`); + console.log(`Indexed ${result.recordCount} bookmarks → ${result.dbPath}`); console.log(formatClassificationSummary(result.summary)); } else { const engine = await resolveEngine(); @@ -781,10 +782,11 @@ export function buildCli() { process.stderr.write('Classifying categories with LLM (batches of 50, ~2 min per batch)...\n'); const catResult = await classifyWithLlm({ engine, + failFast: options.failFast ?? false, onBatch: (done: number, total: number) => { const pct = total > 0 ? Math.round((done / total) * 100) : 0; const elapsed = Math.round((Date.now() - catStart) / 1000); - process.stderr.write(` Categories: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`); + process.stderr.write(` Categories: ${done}/${total} (${pct}%) │ ${elapsed}s elapsed\n`); }, }); console.log(`\nEngine: ${catResult.engine}`); @@ -795,10 +797,11 @@ export function buildCli() { const domResult = await classifyDomainsWithLlm({ engine, all: false, + failFast: options.failFast ?? false, onBatch: (done: number, total: number) => { const pct = total > 0 ? Math.round((done / total) * 100) : 0; const elapsed = Math.round((Date.now() - domStart) / 1000); - process.stderr.write(` Domains: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`); + process.stderr.write(` Domains: ${done}/${total} (${pct}%) │ ${elapsed}s elapsed\n`); }, }); console.log(`\nDomains: ${domResult.classified}/${domResult.totalUnclassified} classified`); @@ -811,6 +814,7 @@ export function buildCli() { .command('classify-domains') .description('Classify bookmarks by subject domain using LLM (ai, finance, etc.)') .option('--all', 'Re-classify all bookmarks, not just missing') + .option('--fail-fast', 'Stop immediately on first classification failure') .action(safe(async (options) => { if (!requireData()) return; const engine = await resolveEngine(); @@ -819,10 +823,11 @@ export function buildCli() { const result = await classifyDomainsWithLlm({ engine, all: options.all ?? false, + failFast: options.failFast ?? false, onBatch: (done: number, total: number) => { const pct = total > 0 ? Math.round((done / total) * 100) : 0; const elapsed = Math.round((Date.now() - start) / 1000); - process.stderr.write(` Domains: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`); + process.stderr.write(` Domains: ${done}/${total} (${pct}%) │ ${elapsed}s elapsed\n`); }, }); console.log(`\nDomains: ${result.classified}/${result.totalUnclassified} classified`); diff --git a/src/engine.ts b/src/engine.ts index f8b35db..8ca9e6f 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -5,7 +5,7 @@ * Remembers the user's choice in ~/.ft-bookmarks/.preferences. */ -import { execFileSync, execFile } from 'node:child_process'; +import { execFileSync, execFile, execSync, type ExecSyncOptions } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { loadPreferences, savePreferences } from './preferences.js'; @@ -150,14 +150,34 @@ export interface InvokeOptions { maxBuffer?: number; } +export interface InvokeError extends Error { + stderr: string; + stdout: string; +} + export function invokeEngine(engine: ResolvedEngine, prompt: string, opts: InvokeOptions = {}): string { const { bin, args } = engine.config; - return execFileSync(bin, args(prompt), { + const fullArgs = args(prompt); + + const result = execSync(`${bin} ${fullArgs.map(a => `'${a.replace(/'/g, "'\\''")}'`).join(' ')}`, { encoding: 'utf-8', timeout: opts.timeout ?? 120_000, maxBuffer: opts.maxBuffer ?? 1024 * 1024, - stdio: ['pipe', 'pipe', 'ignore'], - }).trim(); + } as ExecSyncOptions) as unknown as { stdout: string; stderr: string }; + + const stdout = result.stdout.trim(); + const stderr = result.stderr ?? ''; + + const fullOutput = stdout + '\n' + stderr; + if (fullOutput.includes('ERROR:') || fullOutput.includes('error:')) { + const errorMsg = stderr.includes('ERROR:') || stderr.includes('error:') ? stderr.trim() : stdout.trim(); + const error = new Error(errorMsg) as InvokeError; + error.stderr = stderr; + error.stdout = stdout; + throw error; + } + + return stdout; } /** @@ -171,8 +191,13 @@ export function invokeEngineAsync(engine: ResolvedEngine, prompt: string, opts: encoding: 'utf-8', timeout: opts.timeout ?? 120_000, maxBuffer: opts.maxBuffer ?? 1024 * 1024, - }, (err, stdout) => { - if (err) return reject(err); + }, (err, stdout, stderr) => { + if (err) { + const fullErr = new Error(stderr?.trim() || err.message) as InvokeError; + fullErr.stderr = stderr ?? ''; + fullErr.stdout = stdout?.trim() ?? ''; + return reject(fullErr); + } resolve(stdout.trim()); }); }); From 5018c745da70d2fcbd0f58ca9bd726f20e777445 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Thu, 9 Apr 2026 20:24:15 +1000 Subject: [PATCH 3/4] docs(README): update with all commands --- CLAUDE.md | 12 +++ README.md | 240 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 192 insertions(+), 60 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 91daf6e..41f2aa8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,18 @@ This is the Field Theory CLI — a standalone tool for syncing and querying X/Twitter bookmarks locally. +## Running locally + +**Important:** If you have this installed globally (`npm install -g fieldtheory`), make sure to run the local version for development: + +```bash +pnpm start +# or +npm run start -- +``` + +The global `ft` command may be a different version than the local codebase. + ## Commands ```bash diff --git a/README.md b/README.md index e00827f..8683c52 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,18 @@ Sync and store locally all of your X/Twitter bookmarks. Search, classify, and ma Free and open source. Designed for Mac. +## Running locally + +**Important:** If you have this installed globally (`npm install -g fieldtheory`), make sure to run the local version for development: + +```bash +pnpm start +# or +npm run start -- +``` + +The global `ft` command may be a different version than the local codebase. + ## Install ```bash @@ -31,66 +43,174 @@ On first run, `ft sync` extracts your X session from Chrome and downloads your b ## Commands -### Sync - -| Command | Description | -|---------|-------------| -| `ft sync` | Download and sync bookmarks (no API required) | -| `ft sync --full` | Full history crawl (not just incremental) | -| `ft sync --gaps` | Backfill missing quoted tweets and expand truncated articles | -| `ft sync --classify` | Sync then classify new bookmarks with LLM | -| `ft sync --api` | Sync via OAuth API (cross-platform) | -| `ft auth` | Set up OAuth for API-based sync (optional) | - -### Search and browse - -| Command | Description | -|---------|-------------| -| `ft search ` | Full-text search with BM25 ranking | -| `ft list` | Filter by author, date, category, domain | -| `ft show ` | Show one bookmark in detail | -| `ft sample ` | Random sample from a category | -| `ft stats` | Top authors, languages, date range | -| `ft viz` | Terminal dashboard with sparklines, categories, and domains | -| `ft categories` | Show category distribution | -| `ft domains` | Subject domain distribution | - -### Classification - -| Command | Description | -|---------|-------------| -| `ft classify` | Classify by category and domain using LLM | -| `ft classify --regex` | Classify by category using simple regex | -| `ft classify-domains` | Classify by subject domain only (LLM) | -| `ft model` | View or change the default LLM engine | - -### Knowledge base - -| Command | Description | -|---------|-------------| -| `ft md` | Export bookmarks as individual markdown files | -| `ft wiki` | Compile a Karpathy-style interlinked knowledge base | -| `ft ask ` | Ask questions against the knowledge base | -| `ft ask --save` | Ask and save the answer as a concept page | -| `ft lint` | Health-check the wiki for broken links and missing pages | -| `ft lint --fix` | Auto-fix fixable wiki issues | - -### Agent integration - -| Command | Description | -|---------|-------------| -| `ft skill install` | Install `/fieldtheory` skill for Claude Code and Codex | -| `ft skill show` | Print skill content to stdout | -| `ft skill uninstall` | Remove installed skill files | - -### Utilities - -| Command | Description | -|---------|-------------| -| `ft index` | Rebuild search index from JSONL cache (preserves classifications) | -| `ft fetch-media` | Download media assets (static images only) | -| `ft status` | Show sync status and data location | -| `ft path` | Print data directory path | +### sync +Download and sync bookmarks from X into your local database. + +| Flag | Description | +|------|-------------| +| (default) | Incremental sync from your last bookmark | +| `--rebuild` | Full re-crawl of all bookmarks | +| `--gaps` | Backfill missing data (quoted tweets, truncated articles) | +| `--classify` | Classify new bookmarks with LLM after syncing | +| `--api` | Use OAuth v2 API instead of Chrome session | +| `--yes` | Skip confirmation prompts | +| `--max-pages ` | Max pages to fetch (default: 500) | +| `--target-adds ` | Stop after N new bookmarks | +| `--delay-ms ` | Delay between requests in ms (default: 600) | +| `--max-minutes ` | Max runtime in minutes (default: 30) | +| `--browser ` | Browser to read session from (chrome, chromium, brave, firefox) | +| `--cookies ` | Pass ct0 and auth_token directly (skips browser extraction) | +| `--chrome-user-data-dir ` | Chrome-family user-data directory | +| `--chrome-profile-directory ` | Chrome-family profile name | +| `--firefox-profile-dir ` | Firefox profile directory | + +### search +Full-text search across bookmarks with BM25 ranking. + +| Flag | Description | +|------|-------------| +| `` | Search query (supports FTS5 syntax: AND, OR, NOT, "exact phrase") | +| `--author ` | Filter by author handle | +| `--after ` | Bookmarks posted after this date (YYYY-MM-DD) | +| `--before ` | Bookmarks posted before this date (YYYY-MM-DD) | +| `--limit ` | Max results (default: 20) | + +### list +List bookmarks with filters. + +| Flag | Description | +|------|-------------| +| `--query ` | Text query (FTS5 syntax) | +| `--author ` | Filter by author handle | +| `--after ` | Posted after (YYYY-MM-DD) | +| `--before ` | Posted before (YYYY-MM-DD) | +| `--category ` | Filter by category | +| `--domain ` | Filter by domain | +| `--limit ` | Max results (default: 30) | +| `--offset ` | Offset into results (default: 0) | +| `--json` | JSON output | + +### show +Show one bookmark in detail. + +| Flag | Description | +|------|-------------| +| `` | Bookmark ID | +| `--json` | JSON output | + +### sample +Random sample from a category or domain. + +| Flag | Description | +|------|-------------| +| `` | Category or domain to sample from | +| `--limit ` | Max results (default: 10) | + +### classify +Classify bookmarks by category and domain using LLM. + +| Flag | Description | +|------|-------------| +| (default) | Classify categories and domains with LLM | +| `--regex` | Use simple regex classification instead of LLM | +| `--fail-fast` | Stop immediately on first classification failure | + +### classify-domains +Classify bookmarks by subject domain only (LLM). + +| Flag | Description | +|------|-------------| +| (default) | Classify only missing domains | +| `--all` | Re-classify all bookmarks, not just missing | +| `--fail-fast` | Stop immediately on first classification failure | + +### md +Export bookmarks as individual markdown files. + +| Flag | Description | +|------|-------------| +| `--force` | Re-export all bookmarks (overwrite existing files) | +| `--format ` | Filename format: `rev-iso` (default, e.g. 2024-01-15-id.md) or `legacy` (e.g. id-tweettext.md) | + +### wiki +Compile a Karpathy-style interlinked knowledge base. + +| Flag | Description | +|------|-------------| +| (default) | Incremental: only pages whose source bookmark count changed | +| `--full` | Recompile all pages (ignore incremental cache) | + +### ask +Ask questions against the knowledge base. + +| Flag | Description | +|------|-------------| +| `` | Question to ask | +| `--save` | Save the answer as a concept page | +| `--json` | Output JSON instead of text | + +### lint +Health-check the wiki for broken links and missing pages. + +| Flag | Description | +|------|-------------| +| (default) | Check and report issues | +| `--fix` | Auto-fix fixable issues with targeted recompile | +| `--json` | Output JSON instead of text | + +### index +Rebuild search index from JSONL cache. + +| Flag | Description | +|------|-------------| +| (default) | Preserve existing classifications | +| `--force` | Drop and rebuild from scratch (loses classifications) | + +### fetch-media +Download media assets (static images only). + +| Flag | Description | +|------|-------------| +| `--limit ` | Max bookmarks to process (default: 100) | +| `--max-bytes ` | Per-asset byte limit (default: 50MB) | + +### model +View or change the default LLM engine. + +| Flag | Description | +|------|-------------| +| (default) | Show current model and available options | +| `` | Set engine to `claude` or `codex` | + +### auth +Set up OAuth for API-based sync (optional). + +### status +Show sync status and data location. + +### path +Print data directory path. + +### categories +Show category distribution. + +### domains +Show subject domain distribution. + +### stats +Top authors, languages, date range. + +### viz +Terminal dashboard with sparklines, categories, and domains. + +### skill install +Install `/fieldtheory` skill for Claude Code and Codex. + +### skill show +Print skill content to stdout. + +### skill uninstall +Remove installed skill files. ## Agent integration From 31f71326d3aab57cf638a78fd9ace278acea0da5 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Sat, 11 Apr 2026 03:40:28 +1000 Subject: [PATCH 4/4] fix(engine, viz, md-export): address PR #70 review comments - engine.ts: replace execSync (returns string, not {stdout,stderr}) with spawnSync which properly captures both streams. Removes broken "as unknown as" cast and shell interpolation hack. - bookmarks-viz.ts: define ANSI_RE as regex literal instead of constructing from ESC constant (literal [ opened a character class). - md-export.ts: accept ISO dates with time components by checking dash positions (>=10 length) instead of strict length===10. Addresses Cursor Bugbot comments on afar1/fieldtheory-cli#70. --- src/bookmarks-viz.ts | 4 ++- src/engine.ts | 23 ++++++++---- src/md-export.ts | 16 +++++---- tests/engine-invoke.test.ts | 70 +++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 tests/engine-invoke.test.ts diff --git a/src/bookmarks-viz.ts b/src/bookmarks-viz.ts index e326b9e..2e9f021 100644 --- a/src/bookmarks-viz.ts +++ b/src/bookmarks-viz.ts @@ -4,6 +4,8 @@ import { twitterBookmarksIndexPath } from './paths.js'; // ── ANSI helpers ───────────────────────────────────────────────────────────── export const ESC = '\x1b['; +/** Regex that matches a single ANSI escape sequence (CSI … m). */ +const ANSI_RE = /\x1b\[[^m]*m/g; export const RESET = `${ESC}0m`; const BOLD = `${ESC}1m`; const DIM = `${ESC}2m`; @@ -93,7 +95,7 @@ function boxBottom(width: number): string { return C.dim + '╰' + '─'.repeat(width - 2) + '╯' + RESET; } function boxRow(content: string, width: number): string { - const stripped = content.replace(new RegExp(ESC + '\\[([^m]*m)', 'g'), ''); + const stripped = content.replace(ANSI_RE, ''); const pad = Math.max(0, width - 4 - stripped.length); return C.dim + '│ ' + RESET + content + ' '.repeat(pad) + C.dim + ' │' + RESET; } diff --git a/src/engine.ts b/src/engine.ts index 8ca9e6f..897de9c 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -5,7 +5,7 @@ * Remembers the user's choice in ~/.ft-bookmarks/.preferences. */ -import { execFileSync, execFile, execSync, type ExecSyncOptions } from 'node:child_process'; +import { execFile, spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { loadPreferences, savePreferences } from './preferences.js'; @@ -157,20 +157,31 @@ export interface InvokeError extends Error { export function invokeEngine(engine: ResolvedEngine, prompt: string, opts: InvokeOptions = {}): string { const { bin, args } = engine.config; - const fullArgs = args(prompt); - const result = execSync(`${bin} ${fullArgs.map(a => `'${a.replace(/'/g, "'\\''")}'`).join(' ')}`, { + const result = spawnSync(bin, args(prompt), { encoding: 'utf-8', timeout: opts.timeout ?? 120_000, maxBuffer: opts.maxBuffer ?? 1024 * 1024, - } as ExecSyncOptions) as unknown as { stdout: string; stderr: string }; + }); + + if (result.error) { + throw result.error; + } - const stdout = result.stdout.trim(); + const stdout = (result.stdout ?? '').trim(); const stderr = result.stderr ?? ''; + if (result.status !== 0) { + const errorMsg = stderr.trim() || stdout || `${bin} exited with code ${result.status}`; + const error = new Error(errorMsg) as InvokeError; + error.stderr = stderr; + error.stdout = stdout; + throw error; + } + const fullOutput = stdout + '\n' + stderr; if (fullOutput.includes('ERROR:') || fullOutput.includes('error:')) { - const errorMsg = stderr.includes('ERROR:') || stderr.includes('error:') ? stderr.trim() : stdout.trim(); + const errorMsg = stderr.includes('ERROR:') || stderr.includes('error:') ? stderr.trim() : stdout; const error = new Error(errorMsg) as InvokeError; error.stderr = stderr; error.stdout = stdout; diff --git a/src/md-export.ts b/src/md-export.ts index 36ff6f6..0e320aa 100644 --- a/src/md-export.ts +++ b/src/md-export.ts @@ -52,16 +52,18 @@ function parsePostedAt(dateStr: string | null | undefined): { } | null { if (!dateStr) return null; - if (dateStr.length === 10 && dateStr.includes("-")) { - const year = parseInt(dateStr.slice(0, 4)); - const month = parseInt(dateStr.slice(5, 7)) - 1; - const day = parseInt(dateStr.slice(8, 10)); + // ISO date — bare ("2024-01-15") or with time ("2024-01-15T00:00:00Z") + if (dateStr.length >= 10 && dateStr[4] === '-' && dateStr[7] === '-') { + const iso = dateStr.slice(0, 10); // "YYYY-MM-DD" + const year = parseInt(iso.slice(0, 4)); + const month = parseInt(iso.slice(5, 7)) - 1; + const day = parseInt(iso.slice(8, 10)); const dowNum = new Date(year, month, day).getDay(); const dow = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][dowNum]; return { - year: dateStr.slice(0, 4), - month: dateStr.slice(5, 7), - day: dateStr.slice(8, 10), + year: iso.slice(0, 4), + month: iso.slice(5, 7), + day: iso.slice(8, 10), dow, }; } diff --git a/tests/engine-invoke.test.ts b/tests/engine-invoke.test.ts new file mode 100644 index 0000000..b282c0f --- /dev/null +++ b/tests/engine-invoke.test.ts @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execSync } from 'node:child_process'; + +// ── Verify execSync return type behavior ──────────────────────────────── + +test('execSync with encoding returns string, not {stdout,stderr}', () => { + const result = execSync('echo hello', { encoding: 'utf-8' }); + assert.equal(typeof result, 'string'); + assert.equal(result.trim(), 'hello'); + + // Confirm accessing .stdout on a string returns undefined + assert.equal((result as any).stdout, undefined); + assert.equal((result as any).stderr, undefined); +}); + +test('execSync as unknown as {stdout,stderr} cast does not crash on .trim() of stdout', () => { + const result = execSync('echo hello', { encoding: 'utf-8' }) as unknown as { stdout: string; stderr: string }; + + // This is what invokeEngine does — result.stdout is undefined + assert.equal(result.stdout, undefined); + assert.equal(result.stderr, undefined); + + // This WOULD crash: result.stdout.trim() + assert.throws( + () => result.stdout!.trim(), + TypeError, + 'Accessing .trim() on undefined should throw TypeError', + ); +}); + +test('execSync capturing stderr requires stdio: pipe for all', () => { + // To capture stderr, you need stdio: ['pipe', 'pipe', 'pipe'] + // AND encoding must NOT be set (returns Buffer), or use spawnSync + const result = execSync('echo out >&2; echo err >&2', { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + + // Even with stdio pipe, execSync with encoding still returns a string (stdout only) + // stderr goes to the parent's stderr, not captured in the return value + assert.equal(typeof result, 'string'); +}); + +test('spawnSync can capture both stdout and stderr separately', async () => { + const { spawnSync } = await import('node:child_process'); + const result = spawnSync('sh', ['-c', 'echo good; echo bad >&2']); + + assert.equal(result.stdout.toString().trim(), 'good'); + assert.equal(result.stderr.toString().trim(), 'bad'); + assert.equal(result.status, 0); +}); + +// ── invokeEngine behavior ─────────────────────────────────────────────── + +test('invokeEngine: calls execSync and returns trimmed stdout', async () => { + // We can't easily import invokeEngine without mocking the engine config, + // but we can verify the core pattern the function uses + const output = execSync('echo " result "', { encoding: 'utf-8' }); + assert.equal(output.trim(), 'result'); +}); + +test('invokeEngine: shell command with single quotes in args', () => { + // Verify the shell escaping pattern from invokeEngine + const prompt = "it's a test"; + const escaped = `'${prompt.replace(/'/g, "'\\''")}'`; + const cmd = `echo ${escaped}`; + const result = execSync(cmd, { encoding: 'utf-8' }); + assert.equal(result.trim(), "it's a test"); +});