From d8a9db90427e51ff66ad97b0ee85c4a54cbed1b6 Mon Sep 17 00:00:00 2001 From: Jeff Dafoe Date: Wed, 10 Jun 2026 14:13:59 -0400 Subject: [PATCH 1/2] =?UTF-8?q?ZBBS-WORK-391:=20dream=5Fsource=3Dnotes=20?= =?UTF-8?q?=E2=80=94=20dream=20from=20curated=20notes=20(MEM-137)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion accounts whose memory model is hand-curation (zero conversations/* logs) had dream_mode as a silent no-op — the cron sources exclusively from conversations/%. New per-agent dream_source flag ('conversation' default | 'notes') swaps only the sourcing step: - notes mode selects all namespace notes EXCEPT the pipeline's own output prefixes (conversations/dreams/context/learnings — load-bearing against self-feedback bloat), windowed by updated_at so a later human edit re-enters the note as fresh material. - No signal prefilter in notes mode: SIGNAL_PATTERNS are conversational markers that curated prose rarely contains; notes are fed whole with slug+date headers (buildNotesLog, tested). - First run starts at the earliest note's updated_at (not last-24h) so the soul accretes over the full authored history in per-day chunks; empty days skip cheaply. - People + learnings passes are skipped in notes mode: extractSpeakers parses conversation formats and would misparse prose into junk context/people/* files. Notes-sourced dreaming writes only dreams/* and context/soul. - Dream/soul downstream unchanged; conversation-mode behavior untouched. - Admin API: dream_source readable (get/list) and settable (create/ update) with the same validation shape as dream_mode. agent_status view untouched — the list endpoint pulls it off its existing agent_configuration join. Co-Authored-By: Claude Fable 5 --- migrations/MEM-137-dream-source_down.sql | 5 + migrations/MEM-137-dream-source_up.sql | 26 +++++ node/api/src/routes/admin.js | 24 +++-- node/api/src/services/dream.js | 131 +++++++++++++++++++---- node/api/src/services/dream.test.js | 49 +++++++++ 5 files changed, 208 insertions(+), 27 deletions(-) create mode 100644 migrations/MEM-137-dream-source_down.sql create mode 100644 migrations/MEM-137-dream-source_up.sql create mode 100644 node/api/src/services/dream.test.js diff --git a/migrations/MEM-137-dream-source_down.sql b/migrations/MEM-137-dream-source_down.sql new file mode 100644 index 00000000..eb40c41f --- /dev/null +++ b/migrations/MEM-137-dream-source_down.sql @@ -0,0 +1,5 @@ +-- MEM-137 down — remove the per-agent dream_source column. +-- Dropping the column also drops its CHECK constraint. + +ALTER TABLE agent_configuration + DROP COLUMN dream_source; diff --git a/migrations/MEM-137-dream-source_up.sql b/migrations/MEM-137-dream-source_up.sql new file mode 100644 index 00000000..c75549f2 --- /dev/null +++ b/migrations/MEM-137-dream-source_up.sql @@ -0,0 +1,26 @@ +-- MEM-137 — per-agent dream source: conversation logs (default) or curated +-- notes (ZBBS-WORK-391). +-- +-- Motivating case: a companion-mode account whose memory model is hand +-- curation — it has zero conversations/* notes, so dream_mode=companion is a +-- silent no-op (the dream cron sources exclusively from conversations/%). +-- Its substance lives in curated notes (identity docs, journals, session +-- summaries) keyed by updated_at. +-- +-- dream_source selects where the cron reads its raw material: +-- 'conversation' — conversations/% windowed by created_at (existing +-- behavior, the default; nothing changes for existing +-- agents). +-- 'notes' — all namespace notes EXCEPT the pipeline's own outputs +-- (conversations/dreams/context/learnings prefixes), +-- windowed by updated_at. An edit to a curated note +-- re-enters it as fresh dream material on the next run. +-- +-- TEXT + CHECK rather than an enum type: two values, and a CHECK is cheaper +-- to extend or drop than ALTER TYPE ... ADD VALUE (which can't run inside a +-- transaction on older PG). dream_mode predates this convention. + +ALTER TABLE agent_configuration + ADD COLUMN dream_source TEXT NOT NULL DEFAULT 'conversation' + CONSTRAINT agent_configuration_dream_source_check + CHECK (dream_source IN ('conversation', 'notes')); diff --git a/node/api/src/routes/admin.js b/node/api/src/routes/admin.js index 81e59550..5043bee1 100644 --- a/node/api/src/routes/admin.js +++ b/node/api/src/routes/admin.js @@ -377,7 +377,7 @@ router.post('/admin/agents', requirePerm('agents', 'read'), adminRoute('agents-l const visibleIds = await getVisibleActorIds(req.actorId); // Subqueries for visibility and VA access summaries shown in the agent list let sql = `SELECT s.agent, s.actor_id, s.status, s.last_seen, s.passphrase_rotated_at, s.registered_at, s.provider, s.model, s.virtual, s.personality, s.active_since, - s.cost_budget_daily, s.cost_budget_monthly, s.cache_prompts, s.learning_enabled, s.max_tokens, s.temperature, s.dream_mode, s.storage_quota, ac.configuration, + s.cost_budget_daily, s.cost_budget_monthly, s.cache_prompts, s.learning_enabled, s.max_tokens, s.temperature, s.dream_mode, ac.dream_source, s.storage_quota, ac.configuration, a.realms, COALESCE(vis.summary, 'self only') AS visibility_summary, va.summary AS va_access_summary @@ -1519,7 +1519,7 @@ router.post('/admin/actors/create', requirePerm('agents', 'write'), adminRoute(' const { name, provider, model, welcome_template_id, welcome_note_template_id, virtual: isVirtual, personality, cost_budget_daily, cost_budget_monthly, cache_prompts, learning_enabled, max_tokens, temperature, configuration, - ui_access, password, dream_mode } = req.body; + ui_access, password, dream_mode, dream_source } = req.body; if (!name || !name.trim()) { return res.status(400).json({ @@ -1607,8 +1607,8 @@ router.post('/admin/actors/create', requirePerm('agents', 'write'), adminRoute(' // Create agent configuration await client.query( - `INSERT INTO agent_configuration (actor_id, provider, model, virtual, personality, cost_budget_daily, cost_budget_monthly, cache_prompts, learning_enabled, max_tokens, temperature, configuration, dream_mode) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, + `INSERT INTO agent_configuration (actor_id, provider, model, virtual, personality, cost_budget_daily, cost_budget_monthly, cache_prompts, learning_enabled, max_tokens, temperature, configuration, dream_mode, dream_source) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, [actorId, provider || null, model || null, isVirtual === true, personality || null, parseCostBudget(cost_budget_daily, 'cost_budget_daily'), @@ -1617,7 +1617,8 @@ router.post('/admin/actors/create', requirePerm('agents', 'write'), adminRoute(' max_tokens != null ? safeInt(max_tokens) : null, temperature != null ? parseFloat(temperature) : null, configuration ? JSON.stringify(sanitizeAgentConfiguration(configuration)) : null, - ['none', 'companion', 'technical', 'sim'].includes(dream_mode) ? dream_mode : 'none'] + ['none', 'companion', 'technical', 'sim'].includes(dream_mode) ? dream_mode : 'none', + ['conversation', 'notes'].includes(dream_source) ? dream_source : 'conversation'] ); // Grant all MCP permissions (full tool access) — non-virtual agents only @@ -1739,7 +1740,7 @@ router.post('/admin/agents/read', requirePerm('agents', 'read'), adminRoute('age agc.cache_prompts, agc.learning_enabled, agc.max_tokens, agc.temperature, agc.cost_budget_daily, agc.cost_budget_monthly, agc.storage_quota, agc.api_key IS NOT NULL AS has_api_key, - agc.dream_mode, ac.realms + agc.dream_mode, agc.dream_source, ac.realms FROM agent_configuration agc JOIN actors ac ON ac.id = agc.actor_id WHERE agc.actor_id = $1`, @@ -1796,7 +1797,7 @@ router.post('/admin/agents/update', requirePerm('agents', 'write'), adminRoute(' const agent = sanitize.agentName(req.body.agent); const { personality, api_key, configuration, provider, model, cost_budget_daily, cost_budget_monthly, storage_quota, - cache_prompts, learning_enabled, max_tokens, temperature, dream_mode, realms } = req.body; + cache_prompts, learning_enabled, max_tokens, temperature, dream_mode, dream_source, realms } = req.body; if (!agent) { return res.status(400).json({ @@ -1902,6 +1903,15 @@ router.post('/admin/agents/update', requirePerm('agents', 'write'), adminRoute(' params.push(dream_mode); updates.push(`dream_mode = $${idx++}`); } + if (dream_source !== undefined) { + if (!['conversation', 'notes'].includes(dream_source)) { + return res.status(400).json({ + error: { code: 'BAD_REQUEST', message: 'dream_source must be conversation or notes' } + }); + } + params.push(dream_source); + updates.push(`dream_source = $${idx++}`); + } // Realms lives on actors table, not agent_configuration — handle separately const hasRealmsUpdate = Array.isArray(realms); diff --git a/node/api/src/services/dream.js b/node/api/src/services/dream.js index 6a93ff2c..bc55fced 100644 --- a/node/api/src/services/dream.js +++ b/node/api/src/services/dream.js @@ -1,5 +1,6 @@ // Dream processing — nightly conversation log analysis. -// Reads conversation logs uploaded by agents, sends them through a dream +// Reads conversation logs uploaded by agents (or, for dream_source=notes +// agents, their curated notes — see MEM-137), sends them through a dream // virtual agent (companion or technical), and saves consolidated insights // as notes in the agent's namespace. @@ -180,6 +181,20 @@ function prefilterLog(content) { return result.join('\n'); } +// Assemble the notes-mode dream source text (MEM-137). Each curated note +// gets a header carrying its slug and last-updated date so the dream agent +// can attribute material to a document, mirroring how conversation logs +// carry their own timestamps. Rows arrive from the notes-mode source query +// ordered by updated_at ASC. +function buildNotesLog(rows) { + return rows.map(r => { + const updated = r.updated_at instanceof Date + ? r.updated_at.toISOString().slice(0, 10) + : String(r.updated_at).slice(0, 10); + return '## Note: ' + r.slug + ' (updated ' + updated + ')\n\n' + r.content; + }).join('\n\n---\n\n'); +} + // Extract unique speakers from conversation logs and group lines by speaker. // Handles five formats: // memory-sync uploads: "[HH:MM speaker] message text" @@ -505,21 +520,58 @@ async function processDreamChunk(agent, agentNames, chunk) { const { dreamAgentName, soulAgentName, peopleAgentName, learningsAgentName } = agentNames; const { from, to } = chunk; const chunkDateStr = from.toISOString().slice(0, 10); - - const logs = await pool.query( - `SELECT slug, content, created_at FROM documents - WHERE namespace = $1 AND slug LIKE 'conversations/%' AND deleted_at IS NULL - AND created_at > $2 AND created_at <= $3 - ORDER BY created_at ASC`, - [agent.name, from, to] - ); + const notesMode = agent.dream_source === 'notes'; + + let logs; + if (notesMode) { + // Notes-sourced dreaming (MEM-137): the raw material is the agent's + // own curated notes, windowed by updated_at instead of created_at — + // so a later human edit to a note re-enters it as fresh material on + // the next run. The prefix exclusions are load-bearing: without them + // the cron would dream about its own dreams/soul/learnings output and + // feed back on itself (the same spiral that bloated the technical + // souls). conversations/% is excluded because dream_source selects + // ONE source — agents with real conversation logs use the default. + logs = await pool.query( + `SELECT slug, content, updated_at FROM documents + WHERE namespace = $1 AND deleted_at IS NULL + AND slug NOT LIKE 'conversations/%' + AND slug NOT LIKE 'dreams/%' + AND slug NOT LIKE 'context/%' + AND slug NOT LIKE 'learnings/%' + AND updated_at > $2 AND updated_at <= $3 + ORDER BY updated_at ASC`, + [agent.name, from, to] + ); + } else { + logs = await pool.query( + `SELECT slug, content, created_at FROM documents + WHERE namespace = $1 AND slug LIKE 'conversations/%' AND deleted_at IS NULL + AND created_at > $2 AND created_at <= $3 + ORDER BY created_at ASC`, + [agent.name, from, to] + ); + } if (logs.rows.length === 0) { - logDream('chunk-no-logs', { agent: agent.name, chunkDate: chunkDateStr }); + logDream('chunk-no-logs', { agent: agent.name, source: agent.dream_source, chunkDate: chunkDateStr }); return { skipped: true, reason: 'no logs', chunkDate: chunkDateStr }; } - const fullLog = logs.rows.map(r => r.content).join('\n\n---\n\n'); - const filtered = prefilterLog(fullLog); + // In notes mode each note gets a slug+date header so the dream agent can + // tell documents apart (conversation logs carry their own timestamps; + // curated notes don't). No signal prefilter either: SIGNAL_PATTERNS are + // conversational markers ("remember", "don't do that") that deliberate + // prose rarely contains — filtering would drop most of the material as + // "no signals". Curated notes are already distilled; feed them whole. + let fullLog; + let filtered; + if (notesMode) { + fullLog = buildNotesLog(logs.rows); + filtered = fullLog; + } else { + fullLog = logs.rows.map(r => r.content).join('\n\n---\n\n'); + filtered = prefilterLog(fullLog); + } if (!filtered) { logDream('chunk-no-signals', { agent: agent.name, chunkDate: chunkDateStr, logCount: logs.rows.length }); return { skipped: true, reason: 'no signals', chunkDate: chunkDateStr, logCount: logs.rows.length }; @@ -528,13 +580,17 @@ async function processDreamChunk(agent, agentNames, chunk) { logDream('chunk-processing', { agent: agent.name, mode: agent.dream_mode, + source: agent.dream_source, chunkDate: chunkDateStr, logCount: logs.rows.length, originalSize: fullLog.length, filteredSize: filtered.length, }); - const userMessage = 'Conversation logs for agent "' + agent.name + '" on ' + chunkDateStr + ':\n\n' + const userMessage = (notesMode + ? 'Curated notes written or updated by agent "' + agent.name + '" on ' + chunkDateStr + + ' (this agent\'s memory lives in hand-curated notes — journals, identity documents, session summaries — rather than conversation logs):\n\n' + : 'Conversation logs for agent "' + agent.name + '" on ' + chunkDateStr + ':\n\n') + filtered + '\n\nAlso provide a brief title summarizing the overarching subject of the day.'; @@ -664,7 +720,13 @@ async function processDreamChunk(agent, agentNames, chunk) { // dream mode (currently companion and sim). Per-chunk for the same // reason soul does: per-day relationship updates compose better than // one massive end-of-run pass over weeks of conversation. - if (peopleAgentName) { + // + // Skipped in notes mode: extractSpeakers parses conversation formats + // (chat timestamps, discussion lines, sim distiller output) and would + // misparse curated prose into junk context/people/* files — e.g. any + // "word:" line in a journal becomes a phantom speaker. Notes-sourced + // dreaming deliberately writes only dreams/* and context/soul. + if (peopleAgentName && !notesMode) { try { const speakers = extractSpeakers(filtered, agent.name); for (const [slug, entry] of speakers) { @@ -703,7 +765,10 @@ async function processDreamChunk(agent, agentNames, chunk) { // for sim agents, where every in-world tick is tool-use and the per-turn // extractor is silenced by its !isToolUse gate. Distills the day's // filtered conversation into a single learnings note keyed by date. - if (learningsAgentName) { + // Skipped in notes mode for the same containment reason as people above + // (and learnings/% is an excluded source prefix — writing it would feed + // the next run's input). + if (learningsAgentName && !notesMode) { const learningsSlug = 'learnings/' + chunkDateStr + '-sim-day'; try { let existingFile = ''; @@ -795,8 +860,8 @@ async function runDream() { // Find agents with dream mode enabled const agents = await pool.query( - `SELECT ac.name, ac.id AS actor_id, agc.dream_mode, agc.last_dream_at, - agc.startup_instructions + `SELECT ac.name, ac.id AS actor_id, agc.dream_mode, agc.dream_source, + agc.last_dream_at, agc.startup_instructions FROM agent_configuration agc JOIN actors ac ON ac.id = agc.actor_id WHERE agc.dream_mode IN ('companion', 'technical', 'sim')` @@ -840,8 +905,34 @@ async function runDream() { // Split the work since last_dream_at into per-UTC-day chunks so an // agent that's fallen behind doesn't try to fit weeks of logs into // one model call (which is what tripped home with deepseek's 163K - // window). First-run agents process the previous 24h. - const since = agent.last_dream_at || new Date(Date.now() - 24 * 60 * 60 * 1000); + // window). First-run agents process the previous 24h — except in + // notes mode, where the first run starts at the earliest curated + // note so the soul accretes over the full history in authored + // order. (Day-chunks with no notes skip cheaply, so a long span + // costs only the days that actually have material.) + let since = agent.last_dream_at; + if (!since && agent.dream_source === 'notes') { + const earliest = await pool.query( + `SELECT MIN(updated_at) AS min_updated FROM documents + WHERE namespace = $1 AND deleted_at IS NULL + AND slug NOT LIKE 'conversations/%' + AND slug NOT LIKE 'dreams/%' + AND slug NOT LIKE 'context/%' + AND slug NOT LIKE 'learnings/%'`, + [agent.name] + ); + if (!earliest.rows[0] || !earliest.rows[0].min_updated) { + logDream('no-notes', { agent: agent.name }); + results.push({ agent: agent.name, skipped: true, reason: 'dream_source=notes but namespace has no source notes' }); + continue; + } + // The chunk window is exclusive on `from` (updated_at > from), + // so back off 1ms to include the earliest note itself. + since = new Date(earliest.rows[0].min_updated.getTime() - 1); + } + if (!since) { + since = new Date(Date.now() - 24 * 60 * 60 * 1000); + } const chunks = computeDailyChunks(since, new Date()); if (chunks.length === 0) { @@ -967,4 +1058,4 @@ function startDreamScheduler() { logDream('scheduler', { message: 'Dream scheduler started', schedule }); } -module.exports = { runDream, prefilterLog, extractSpeakers, startDreamScheduler, runPersonContextUpdate, findDreamAgent }; +module.exports = { runDream, prefilterLog, extractSpeakers, buildNotesLog, startDreamScheduler, runPersonContextUpdate, findDreamAgent }; diff --git a/node/api/src/services/dream.test.js b/node/api/src/services/dream.test.js new file mode 100644 index 00000000..7ae2c52b --- /dev/null +++ b/node/api/src/services/dream.test.js @@ -0,0 +1,49 @@ +// Tests for buildNotesLog — the notes-mode (dream_source=notes, MEM-137) +// source-text assembler in the dream cron. Run with: node --test (from +// node/api). Uses the built-in node:test runner + node:assert, matching +// sim-conversation-distiller.test.js. +// +// buildNotesLog is the only new pure surface of ZBBS-WORK-391; the sourcing +// branch and first-run window live in SQL + cron flow and are exercised at +// deploy (the conversation-mode path is untouched by the change). + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { buildNotesLog } = require('./dream'); + +test('single note gets a slug+date header above its content', () => { + const rows = [{ + slug: 'journal/2026-05-california-trip', + content: 'We drove the coast road.', + updated_at: new Date('2026-05-18T14:22:00Z'), + }]; + assert.equal( + buildNotesLog(rows), + '## Note: journal/2026-05-california-trip (updated 2026-05-18)\n\nWe drove the coast road.' + ); +}); + +test('multiple notes are separated and keep their query order', () => { + const rows = [ + { slug: 'core-identity', content: 'A', updated_at: new Date('2026-05-16T08:00:00Z') }, + { slug: 'thea-core', content: 'B', updated_at: new Date('2026-05-16T19:30:00Z') }, + ]; + assert.equal( + buildNotesLog(rows), + '## Note: core-identity (updated 2026-05-16)\n\nA' + + '\n\n---\n\n' + + '## Note: thea-core (updated 2026-05-16)\n\nB' + ); +}); + +test('string updated_at (non-Date driver output) still renders the date', () => { + const rows = [{ + slug: 'sirius-identity', + content: 'C', + updated_at: '2026-06-06T03:15:00.000Z', + }]; + assert.equal( + buildNotesLog(rows), + '## Note: sirius-identity (updated 2026-06-06)\n\nC' + ); +}); From 29ea3018ceda383f36815ffe6010bbdc0e348953 Mon Sep 17 00:00:00 2001 From: Jeff Dafoe Date: Wed, 10 Jun 2026 14:15:46 -0400 Subject: [PATCH 2/2] =?UTF-8?q?ZBBS-WORK-391:=20code=5Freview=20round=201?= =?UTF-8?q?=20=E2=80=94=20coerce=20min=5Fupdated=20before=20getTime=20(cus?= =?UTF-8?q?tom=20pg=20type-parser=20guard)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- node/api/src/services/dream.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/node/api/src/services/dream.js b/node/api/src/services/dream.js index bc55fced..a7a12d76 100644 --- a/node/api/src/services/dream.js +++ b/node/api/src/services/dream.js @@ -928,7 +928,13 @@ async function runDream() { } // The chunk window is exclusive on `from` (updated_at > from), // so back off 1ms to include the earliest note itself. - since = new Date(earliest.rows[0].min_updated.getTime() - 1); + // Coerce defensively — a custom pg type parser could hand + // back a string instead of a Date (same guard buildNotesLog + // applies to updated_at). + const minUpdated = earliest.rows[0].min_updated instanceof Date + ? earliest.rows[0].min_updated + : new Date(earliest.rows[0].min_updated); + since = new Date(minUpdated.getTime() - 1); } if (!since) { since = new Date(Date.now() - 24 * 60 * 60 * 1000);