feat: add recall export with JSON, Markdown, SQL dump, and SQLite backup formats

CHANGELOG.md

@@ -12,6 +12,11 @@ while MCP tool names (`memory_search`, `memory_add`, etc.) remain stable.
 
 ### Added
 
+- **`recall export`** — portable and disaster-recovery exports (#43): JSON,
+  Markdown, SQL dump, and SQLite (`VACUUM INTO`) formats with a manifest
+  (counts + provenance counts including explicit `unknown`), a stdout/file/
+  directory output contract, and `--backup` writing timestamped, never-
+  overwritten SQL dumps to `~/.agents/Recall/backups/`.
 - Added macOS-primary GitHub Actions CI with Ubuntu portability smoke coverage
   and deterministic release/tag version consistency checks.
 

docs/cli-reference.md

@@ -252,6 +252,54 @@ Suite B (token efficiency) compares the v2 wake-up bundle against v1 and the
 CLAUDE.md baseline. Results are written to `benchmarks/results/` as JSONL plus
 a human-readable `.md` alongside. See `benchmarks/README.md` for methodology.
 
+## Export & Backup
+
+Portable and disaster-recovery exports of the memory database.
+
+```bash
+recall export                                   # JSON export to stdout (summary on stderr)
+recall export --format markdown                 # Human-readable Markdown export
+recall export --format sql --output dump.sql    # Textual SQL dump (schema + INSERTs)
+recall export --format sqlite --output copy.db  # Full database backup (VACUUM INTO)
+recall export --output exports/                 # Directory mode: artifact + manifest.json
+recall export --backup                          # Timestamped SQL dump to ~/.agents/Recall/backups/
+```
+
+Formats:
+
+- **json / markdown** — app-level export of the durable memory tables
+  (`sessions`, `messages`, `decisions`, `learnings`, `breadcrumbs`,
+  `loa_entries`). Every row of a provenance-bearing table carries an explicit
+  `provenance` field; legacy `NULL` provenance is exported as the literal
+  `unknown` — never omitted, never guessed (see Record Provenance above).
+  Embeddings are excluded.
+- **sql** — textual SQL dump (CREATE TABLE + INSERT statements) of the same
+  durable tables. Restorable into an empty database; one-command restore is
+  intentionally not provided.
+- **sqlite** — binary database backup via `VACUUM INTO`: the full DB including
+  embeddings and internal tables. Always requires `--output`.
+
+Output contract:
+
+- No `--output`: export data goes to stdout, the manifest summary to stderr —
+  stdout stays clean for piping.
+- `--output <file>`: writes a single export file and prints the manifest
+  summary to stdout.
+- `--output <dir>`: writes the export artifact plus `manifest.json` into the
+  directory. Directory exports always write `manifest.json`. A path that does
+  not exist yet is treated as a file — add a trailing slash (`exports/`) to
+  request a new directory.
+
+The manifest records the Recall version, timestamp, schema version
+(`PRAGMA user_version`), format, included tables, row counts per table,
+provenance counts per table (including `unknown`), and whether embeddings were
+included.
+
+`recall export --backup` writes a timestamped SQL dump to
+`~/.agents/Recall/backups/` (creating the directory if needed), never
+overwrites an existing file (a `-N` suffix is added on collision), and prints
+the output path.
+
 ## Admin
 
 ```bash

src/commands/export.ts

@@ -0,0 +1,181 @@
+// recall export — portable and disaster-recovery exports (issue #43).
+//
+// Formats:
+// - json / markdown: app-level export of the durable memory tables with an
+//   explicit provenance field per provenance-bearing row (NULL → 'unknown').
+//   Embeddings are excluded.
+// - sql: textual SQL dump (schema + INSERTs) of the durable tables.
+// - sqlite: database backup via VACUUM INTO — full DB including embeddings
+//   and internal tables. Binary, so it always requires --output.
+//
+// Output contract:
+// - no --output: export data goes to stdout, manifest summary to stderr —
+//   stdout stays clean for piping.
+// - --output <file>: data written to the file, manifest summary to stdout.
+// - --output <dir>: artifact plus manifest.json written into the directory,
+//   manifest summary to stdout. Directory exports always write manifest.json.
+// - --backup: timestamped SQL dump into ~/.agents/Recall/backups/ (created if
+//   needed), never overwrites an existing file, prints the output path.
+
+import { existsSync, mkdirSync, statSync, unlinkSync, writeFileSync } from 'fs';
+import { dirname, join, resolve } from 'path';
+import { Database } from 'bun:sqlite';
+import { getDb } from '../db/connection.js';
+import {
+  EXPORT_FORMATS,
+  EXPORT_TABLES,
+  buildManifest,
+  collectExportData,
+  defaultBackupDir,
+  listPhysicalTables,
+  renderJsonExport,
+  renderMarkdownExport,
+  renderSqlDump,
+  resolveNonClobbering,
+  timestampSlug,
+  writeNonClobbering,
+  type ExportFormat,
+  type ExportManifest,
+} from '../lib/export.js';
+
+export interface ExportOptions {
+  format?: string;
+  output?: string;
+  backup?: boolean;
+  /** Test seam — the CLI always uses defaultBackupDir(). */
+  backupDir?: string;
+  /** Test seam — the CLI always uses the current time. */
+  now?: Date;
+}
+
+// sqlite takes its own branch in runExport and never reaches this map.
+const ARTIFACT_EXT: Record<'json' | 'markdown' | 'sql', string> = {
+  json: 'json',
+  markdown: 'md',
+  sql: 'sql',
+};
+
+function manifestSummary(manifest: ExportManifest, paths: string[]): string {
+  const lines: string[] = [];
+  lines.push(`Recall export — format: ${manifest.format}`);
+  for (const p of paths) lines.push(`  Output: ${p}`);
+  lines.push(`  Recall ${manifest.recall_version} | schema v${manifest.schema_version} | ${manifest.created_at}`);
+  lines.push(`  Rows: ${manifest.tables.map(t => `${t}=${manifest.counts[t]}`).join(', ')}`);
+  const provenance = Object.entries(manifest.provenance_counts)
+    .map(([table, hist]) => `${table} { ${Object.entries(hist).map(([k, v]) => `${k}=${v}`).join(', ')} }`)
+    .join('; ');
+  lines.push(`  Provenance: ${provenance}`);
+  lines.push(`  Embeddings included: ${manifest.includes_embeddings ? 'yes' : 'no'}`);
+  return lines.join('\n');
+}
+
+function renderTextExport(db: Database, format: 'json' | 'markdown' | 'sql', createdAt: Date): { content: string; manifest: ExportManifest } {
+  const manifest = buildManifest(db, format, [...EXPORT_TABLES], { includesEmbeddings: false, createdAt });
+  if (format === 'sql') {
+    return { content: renderSqlDump(db, manifest), manifest };
+  }
+  const data = collectExportData(db);
+  const content = format === 'json'
+    ? renderJsonExport(manifest, data)
+    : renderMarkdownExport(manifest, data);
+  return { content, manifest };
+}
+
+function runBackup(options: ExportOptions): void {
+  if (options.output) {
+    throw new Error('--backup writes to the backup directory; do not combine it with --output');
+  }
+  if (options.format && options.format !== 'sql') {
+    throw new Error(`--backup always writes a SQL dump; do not combine it with --format ${options.format}`);
+  }
+  const db = getDb();
+  const now = options.now ?? new Date();
+  const dir = options.backupDir ?? defaultBackupDir();
+  mkdirSync(dir, { recursive: true });
+  const { content, manifest } = renderTextExport(db, 'sql', now);
+  const target = writeNonClobbering(join(dir, `recall-backup-${timestampSlug(now)}.sql`), content);
+  console.log(manifestSummary(manifest, [target]));
+}
+
+export function runExport(options: ExportOptions): void {
+  if (options.backup) {
+    runBackup(options);
+    return;
+  }
+
+  const format = (options.format ?? 'json') as ExportFormat;
+  if (!(EXPORT_FORMATS as readonly string[]).includes(format)) {
+    throw new Error(`Unknown format '${options.format}'. Supported: ${EXPORT_FORMATS.join(', ')}`);
+  }
+
+  const db = getDb();
+  const now = options.now ?? new Date();
+
+  // Resolve output mode: none (stdout), file, or directory.
+  const output = options.output ? resolve(options.output) : undefined;
+  const isDir = options.output !== undefined
+    && (options.output.endsWith('/') || (existsSync(output!) && statSync(output!).isDirectory()));
+
+  if (format === 'sqlite') {
+    if (!output) {
+      throw new Error('--format sqlite produces a binary database backup and requires --output <file-or-dir>');
+    }
+    // Manifest first — VACUUM INTO copies the live DB, so counts match it.
+    const manifest = buildManifest(db, 'sqlite', listPhysicalTables(db), { includesEmbeddings: true, createdAt: now });
+    let target: string;
+    if (isDir) {
+      mkdirSync(output, { recursive: true });
+      target = resolveNonClobbering(join(output, `recall-export-${timestampSlug(now)}.db`));
+    } else {
+      if (existsSync(output)) {
+        throw new Error(`Refusing to overwrite existing database backup at ${output}`);
+      }
+      mkdirSync(dirname(output), { recursive: true });
+      target = output;
+    }
+    try {
+      db.prepare('VACUUM INTO ?').run(target);
+    } catch (err) {
+      // SQLite documents that an interrupted VACUUM INTO can leave a partial
+      // output file the application must delete. Without this cleanup a
+      // retry hits the refuse-to-overwrite guard on a corrupt remnant.
+      try { unlinkSync(target); } catch { /* nothing was created */ }
+      throw err;
+    }
+    const paths = [target];
+    if (isDir) {
+      // Manifest written only after the backup exists — a failed VACUUM must
+      // not leave a manifest describing a backup that was never created.
+      const manifestPath = join(output, 'manifest.json');
+      writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+      paths.push(manifestPath);
+    }
+    console.log(manifestSummary(manifest, paths));
+    return;
+  }
+
+  const { content, manifest } = renderTextExport(db, format, now);
+
+  if (!output) {
+    // Piping contract: stdout carries only the export data.
+    process.stdout.write(content);
+    console.error(manifestSummary(manifest, []));
+    return;
+  }
+
+  if (isDir) {
+    mkdirSync(output, { recursive: true });
+    const artifact = writeNonClobbering(
+      join(output, `recall-export-${timestampSlug(now)}.${ARTIFACT_EXT[format]}`),
+      content
+    );
+    const manifestPath = join(output, 'manifest.json');
+    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+    console.log(manifestSummary(manifest, [artifact, manifestPath]));
+    return;
+  }
+
+  mkdirSync(dirname(output), { recursive: true });
+  writeFileSync(output, content);
+  console.log(manifestSummary(manifest, [output]));
+}

src/commands/provenance.ts

@@ -31,9 +31,9 @@
 // variables — bulk UPDATEs with literal predicates — so no chunking applies.
 
 import { getDb } from '../db/connection.js';
+import { PROVENANCE_TABLES } from '../types/index.js';
 
-const BACKFILL_TABLES = ['messages', 'decisions', 'learnings', 'breadcrumbs', 'loa_entries'] as const;
-type BackfillTable = typeof BACKFILL_TABLES[number];
+type BackfillTable = typeof PROVENANCE_TABLES[number];
 
 export interface ProvenanceBackfillOptions {
   dryRun?: boolean;
@@ -93,8 +93,8 @@ export function runProvenanceBackfill(options: ProvenanceBackfillOptions = {}):
   const dryRun = options.dryRun ?? true;
   const target = options.table ?? 'all';
 
-  if (target !== 'all' && !(BACKFILL_TABLES as readonly string[]).includes(target)) {
-    console.error(`Unknown table: ${target}. Use one of: ${BACKFILL_TABLES.join(', ')}, all`);
+  if (target !== 'all' && !(PROVENANCE_TABLES as readonly string[]).includes(target)) {
+    console.error(`Unknown table: ${target}. Use one of: ${PROVENANCE_TABLES.join(', ')}, all`);
     process.exitCode = 1;
     return [];
   }

src/index.ts

@@ -29,6 +29,7 @@ import { runBenchmark, listBenchmarks, reportLatestBenchmark } from './commands/
 import { runOnboard } from './commands/onboard.js';
 import { runMigrate } from './commands/migrate.js';
 import { runPath } from './commands/path.js';
+import { runExport } from './commands/export.js';
 import { closeDb } from './db/connection.js';
 
 const program = new Command();
@@ -634,6 +635,29 @@ program
     closeDb();
   });
 
+// recall export — portable + disaster-recovery exports (issue #43)
+// --format has no Commander default: runExport defaults to json, and --backup
+// must be able to tell an explicitly passed --format from an omitted one.
+program
+  .command('export')
+  .description('Export memory to JSON, Markdown, SQL dump, or SQLite backup')
+  .option('-f, --format <format>', 'Format: json, markdown, sql, sqlite (default: json)')
+  .option('-o, --output <path>', 'Output file or directory (directory exports also write manifest.json)')
+  .option('--backup', 'Write a timestamped SQL dump to ~/.agents/Recall/backups/ (never overwrites)')
+  .action((options) => {
+    try {
+      runExport({
+        format: options.format,
+        output: options.output,
+        backup: options.backup
+      });
+    } catch (err) {
+      console.error(`Export failed: ${err instanceof Error ? err.message : String(err)}`);
+      process.exitCode = 1;
+    }
+    closeDb();
+  });
+
 // Default command: recall <query> → hybrid search (Phase 3: best of both worlds)
 program
   .arguments('[query]')
@@ -644,7 +668,7 @@ program
   .option('-k, --keyword', 'Use keyword search only (FTS5)')
   .option('-v, --vector', 'Use vector search only (semantic)')
   .action(async (query, options) => {
-    if (query && !['init', 'add', 'search', 'recent', 'show', 'stats', 'import', 'import-conversations', 'loa', 'telos', 'docs', 'dump', 'embed', 'semantic', 'hybrid', 'doctor', 'importance', 'provenance', 'pin', 'unpin', 'decision', 'prune', 'cluster', 'import-legacy', 'benchmark', 'onboard', 'migrate', 'path'].includes(query)) {
+    if (query && !['init', 'add', 'search', 'recent', 'show', 'stats', 'import', 'import-conversations', 'loa', 'telos', 'docs', 'dump', 'embed', 'semantic', 'hybrid', 'doctor', 'importance', 'provenance', 'pin', 'unpin', 'decision', 'prune', 'cluster', 'import-legacy', 'benchmark', 'onboard', 'migrate', 'path', 'export'].includes(query)) {
       if (options.keyword) {
         // FTS5 only
         runSearch(query, {