diff --git a/docs/superpowers/plans/2026-04-11-graph-editing-sdk.md b/docs/superpowers/plans/2026-04-11-graph-editing-sdk.md index 3ff94a4..c7f851f 100644 --- a/docs/superpowers/plans/2026-04-11-graph-editing-sdk.md +++ b/docs/superpowers/plans/2026-04-11-graph-editing-sdk.md @@ -13,6 +13,7 @@ ### Task 1: Create Node — Schema + Lib + Route + SDK **Files:** + - Modify: `src/lib/schemas/node.ts` — add createNode request/response schemas - Modify: `src/lib/node.ts` — add `createNode` function - Create: `src/routes/node/create.post.ts` @@ -63,7 +64,12 @@ export async function createNode( nodeType: NodeType, label: string, description?: string, -): Promise<{ id: TypeId<"node">; nodeType: NodeType; label: string; description: string | null }> { +): Promise<{ + id: TypeId<"node">; + nodeType: NodeType; + label: string; + description: string | null; +}> { const db = await useDatabase(); await ensureUser(db, userId); @@ -150,6 +156,7 @@ Run: `pnpm run build` ### Task 2: Edge Schemas + Lib **Files:** + - Create: `src/lib/schemas/edge.ts` — all edge CRUD schemas - Create: `src/lib/edge.ts` — createEdge, deleteEdge, updateEdge business logic @@ -220,7 +227,6 @@ export type UpdateEdgeResponse = z.infer; ```ts /** Edge operations: create, delete, update. */ - import { and, eq, inArray } from "drizzle-orm"; import { nodes, nodeMetadata, edges, edgeEmbeddings } from "~/db/schema"; import { generateEmbeddings } from "~/lib/embeddings"; @@ -278,7 +284,9 @@ export async function createEdge( }> { const db = await useDatabase(); - if (!(await validateNodeOwnership(db, userId, [sourceNodeId, targetNodeId]))) { + if ( + !(await validateNodeOwnership(db, userId, [sourceNodeId, targetNodeId])) + ) { throw new Error("One or both nodes not found"); } @@ -387,12 +395,16 @@ export async function updateEdge( const newSourceNodeId = updates.sourceNodeId ?? current.sourceNodeId; const newTargetNodeId = updates.targetNodeId ?? current.targetNodeId; const newEdgeType = updates.edgeType ?? current.edgeType; - const newDescription = updates.description !== undefined ? updates.description : current.description; + const newDescription = + updates.description !== undefined + ? updates.description + : current.description; // Build update set (only changed fields) const updateSet: Record = {}; if (updates.edgeType) updateSet.edgeType = updates.edgeType; - if (updates.description !== undefined) updateSet.description = updates.description; + if (updates.description !== undefined) + updateSet.description = updates.description; if (updates.sourceNodeId) updateSet.sourceNodeId = updates.sourceNodeId; if (updates.targetNodeId) updateSet.targetNodeId = updates.targetNodeId; @@ -443,6 +455,7 @@ Run: `pnpm run build` ### Task 3: Edge Routes **Files:** + - Create: `src/routes/edge/create.post.ts` - Create: `src/routes/edge/delete.post.ts` - Create: `src/routes/edge/update.post.ts` @@ -542,6 +555,7 @@ Run: `pnpm run build` ### Task 4: Edge SDK Methods + Re-exports **Files:** + - Modify: `src/sdk/memory-client.ts` — add edge methods - Modify: `src/sdk/index.ts` — re-export edge schemas @@ -611,6 +625,7 @@ Run: `pnpm run build` ### Task 5: Merge Nodes — Schema + Lib + Route + SDK **Files:** + - Create: `src/lib/schemas/node-merge.ts` - Modify: `src/lib/node.ts` — add `mergeNodes` function - Create: `src/routes/node/merge.post.ts` @@ -662,7 +677,12 @@ export async function mergeNodes( userId: string, nodeIds: TypeId<"node">[], overrides?: { targetLabel?: string; targetDescription?: string }, -): Promise<{ id: TypeId<"node">; nodeType: string; label: string; description: string | null } | null> { +): Promise<{ + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; +} | null> { const db = await useDatabase(); // Validate all nodes belong to userId @@ -684,9 +704,10 @@ export async function mergeNodes( const survivorRow = foundNodes.find((n) => n.id === survivorId)!; const finalLabel = overrides?.targetLabel ?? survivorRow.label ?? ""; - const finalDescription = overrides?.targetDescription !== undefined - ? overrides.targetDescription - : survivorRow.description; + const finalDescription = + overrides?.targetDescription !== undefined + ? overrides.targetDescription + : survivorRow.description; // Re-point edges from consumed nodes to survivor, dropping duplicates for (const consumedId of consumedIds) { @@ -744,9 +765,7 @@ export async function mergeNodes( `); // Delete remaining duplicate source_links - await db - .delete(sourceLinks) - .where(eq(sourceLinks.nodeId, consumedId)); + await db.delete(sourceLinks).where(eq(sourceLinks.nodeId, consumedId)); } // Delete consumed nodes (cascade handles metadata, embeddings) @@ -758,12 +777,7 @@ export async function mergeNodes( await db .update(nodeMetadata) .set({ label: finalLabel, description: finalDescription }) - .where( - eq( - nodeMetadata.nodeId, - survivorId, - ), - ); + .where(eq(nodeMetadata.nodeId, survivorId)); // Re-generate survivor embedding const embText = `${finalLabel}: ${finalDescription ?? ""}`; @@ -775,7 +789,9 @@ export async function mergeNodes( }); const embedding = embResponse.data[0]?.embedding; if (embedding) { - await db.delete(nodeEmbeddings).where(eq(nodeEmbeddings.nodeId, survivorId)); + await db + .delete(nodeEmbeddings) + .where(eq(nodeEmbeddings.nodeId, survivorId)); await db.insert(nodeEmbeddings).values({ nodeId: survivorId, embedding, @@ -866,6 +882,7 @@ Run: `pnpm run build` ### Task 6: P1 — Update Node with nodeType + Get Atlas Node IDs **Files:** + - Modify: `src/lib/schemas/node.ts` — add `nodeType` to update schema - Modify: `src/lib/node.ts` — update `updateNode` to handle nodeType - Modify: `src/routes/node/update.post.ts` — pass nodeType @@ -903,15 +920,15 @@ export async function updateNode( Add after the ownership check, before the metadata update: ```ts - // Update node type if provided - if (updates.nodeType) { - await db - .update(nodes) - .set({ nodeType: updates.nodeType }) - .where(eq(nodes.id, nodeId)); - } +// Update node type if provided +if (updates.nodeType) { + await db + .update(nodes) + .set({ nodeType: updates.nodeType }) + .where(eq(nodes.id, nodeId)); +} - const effectiveNodeType = updates.nodeType ?? row.nodeType; +const effectiveNodeType = updates.nodeType ?? row.nodeType; ``` And change the return to use `effectiveNodeType`: @@ -933,7 +950,11 @@ Change the destructuring to include `nodeType` and pass it: export default defineEventHandler(async (event) => { const { userId, nodeId, label, description, nodeType } = updateNodeRequestSchema.parse(await readBody(event)); - const result = await updateNode(userId, nodeId, { label, description, nodeType }); + const result = await updateNode(userId, nodeId, { + label, + description, + nodeType, + }); if (!result) { throw createError({ statusCode: 404, statusMessage: "Node not found" }); } @@ -955,21 +976,25 @@ export const queryAtlasNodesResponseSchema = z.object({ nodeIds: z.array(z.string()), }); -export type QueryAtlasNodesRequest = z.infer; -export type QueryAtlasNodesResponse = z.infer; +export type QueryAtlasNodesRequest = z.infer< + typeof queryAtlasNodesRequestSchema +>; +export type QueryAtlasNodesResponse = z.infer< + typeof queryAtlasNodesResponseSchema +>; ``` - [ ] **Step 5: Create route `src/routes/query/atlas-nodes.ts`** ```ts +import { and, eq, or } from "drizzle-orm"; import { defineEventHandler } from "h3"; +import { edges } from "~/db/schema"; import { ensureAssistantAtlasNode } from "~/lib/atlas"; import { queryAtlasNodesRequestSchema, queryAtlasNodesResponseSchema, } from "~/lib/schemas/query-atlas-nodes"; -import { and, eq, or } from "drizzle-orm"; -import { edges } from "~/db/schema"; import { useDatabase } from "~/utils/db"; export default defineEventHandler(async (event) => { @@ -1050,6 +1075,7 @@ Run: `pnpm run build` ### Task 7: P1 — Batch Delete Nodes **Files:** + - Create: `src/lib/schemas/node-batch-delete.ts` - Modify: `src/lib/node.ts` — add `batchDeleteNodes` - Create: `src/routes/node/batch-delete.post.ts` @@ -1072,8 +1098,12 @@ export const batchDeleteNodesResponseSchema = z.object({ count: z.number().int().nonnegative(), }); -export type BatchDeleteNodesRequest = z.infer; -export type BatchDeleteNodesResponse = z.infer; +export type BatchDeleteNodesRequest = z.infer< + typeof batchDeleteNodesRequestSchema +>; +export type BatchDeleteNodesResponse = z.infer< + typeof batchDeleteNodesResponseSchema +>; ``` - [ ] **Step 2: Add `batchDeleteNodes` to `src/lib/node.ts`** @@ -1154,6 +1184,7 @@ Run: `pnpm run build` ### Task 8: P2 — Query Graph nodeTypes Filter **Files:** + - Modify: `src/lib/schemas/query-graph.ts` — add `nodeTypes` field - Modify: `src/lib/query/graph.ts` — apply filter @@ -1206,10 +1237,10 @@ In the query-based branch, pass `nodeTypes` as `excludeNodeTypes` inverted — a After `const seeds = ...`, add a filter only if `nodeTypes` is specified: ```ts - // Apply nodeTypes filter to seed results if specified - const filteredSeeds = params.nodeTypes?.length - ? seeds.filter((s) => params.nodeTypes!.includes(s.type)) - : seeds; +// Apply nodeTypes filter to seed results if specified +const filteredSeeds = params.nodeTypes?.length + ? seeds.filter((s) => params.nodeTypes!.includes(s.type)) + : seeds; ``` Then use `filteredSeeds` instead of `seeds` for the rest of the function. @@ -1223,6 +1254,7 @@ Run: `pnpm run build` ### Task 9: P2 — Node Neighborhood **Files:** + - Create: `src/lib/schemas/node-neighborhood.ts` - Modify: `src/lib/node.ts` — add `getNodeNeighborhood` - Create: `src/routes/node/neighborhood.post.ts` @@ -1247,8 +1279,12 @@ export const nodeNeighborhoodResponseSchema = z.object({ edges: z.array(queryGraphEdgeSchema), }); -export type NodeNeighborhoodRequest = z.infer; -export type NodeNeighborhoodResponse = z.infer; +export type NodeNeighborhoodRequest = z.infer< + typeof nodeNeighborhoodRequestSchema +>; +export type NodeNeighborhoodResponse = z.infer< + typeof nodeNeighborhoodResponseSchema +>; ``` - [ ] **Step 2: Add `getNodeNeighborhood` to `src/lib/node.ts`** @@ -1256,7 +1292,11 @@ export type NodeNeighborhoodResponse = z.infer, depth: 1 | 2 = 1, ): Promise<{ - nodes: { id: TypeId<"node">; nodeType: string; label: string; description: string | null; sourceIds: string[] }[]; - edges: { source: TypeId<"node">; target: TypeId<"node">; edgeType: string; description: string | null }[]; + nodes: { + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; + sourceIds: string[]; + }[]; + edges: { + source: TypeId<"node">; + target: TypeId<"node">; + edgeType: string; + description: string | null; + }[]; } | null> { const db = await useDatabase(); @@ -1291,7 +1342,15 @@ export async function getNodeNeighborhood( if (!focal) return null; const allNodeIds = new Set>([nodeId]); - const nodeMap = new Map, { id: TypeId<"node">; nodeType: string; label: string; description: string | null }>(); + const nodeMap = new Map< + TypeId<"node">, + { + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; + } + >(); nodeMap.set(nodeId, { id: focal.id, nodeType: focal.nodeType, diff --git a/docs/superpowers/specs/2026-04-11-graph-editing-sdk-design.md b/docs/superpowers/specs/2026-04-11-graph-editing-sdk-design.md index 4e4f00f..34b4bb5 100644 --- a/docs/superpowers/specs/2026-04-11-graph-editing-sdk-design.md +++ b/docs/superpowers/specs/2026-04-11-graph-editing-sdk-design.md @@ -26,6 +26,7 @@ No new abstractions. `_fetch` + Zod validation handles everything. Create a memory node directly with a known type and label. **Request:** + ```ts { userId: string @@ -36,6 +37,7 @@ Create a memory node directly with a known type and label. ``` **Response:** + ```ts { node: { id: TypeId<"node">, nodeType: NodeType, label: string, description: string | null } @@ -43,6 +45,7 @@ Create a memory node directly with a known type and label. ``` **Behavior:** + - Creates `nodes` row + `nodeMetadata` row - Generates embedding from `"${label}: ${description ?? ''}"` - Returns created node so frontend can select it immediately @@ -52,6 +55,7 @@ Create a memory node directly with a known type and label. Create a typed edge between two existing nodes. **Request:** + ```ts { userId: string @@ -63,6 +67,7 @@ Create a typed edge between two existing nodes. ``` **Response:** + ```ts { edge: { id: TypeId<"edge">, sourceNodeId, targetNodeId, edgeType, description: string | null } @@ -70,6 +75,7 @@ Create a typed edge between two existing nodes. ``` **Behavior:** + - Validates both nodes exist and belong to userId; 404 if not - Creates `edges` row - Generates edge embedding from `"${sourceLabel} ${edgeType} ${targetLabel}: ${description ?? ''}"` @@ -80,19 +86,24 @@ Create a typed edge between two existing nodes. Delete a single edge by ID. **Request:** + ```ts { - userId: string - edgeId: TypeId<"edge"> + userId: string; + edgeId: TypeId<"edge">; } ``` **Response:** + ```ts -{ deleted: true } +{ + deleted: true; +} ``` **Behavior:** + - Validates edge belongs to userId - Deletes edge (cascade handles edge_embeddings) - Returns 404 if not found @@ -102,6 +113,7 @@ Delete a single edge by ID. Update an edge's type, description, or endpoints. **Request:** + ```ts { userId: string @@ -114,6 +126,7 @@ Update an edge's type, description, or endpoints. ``` **Response:** + ```ts { edge: { id: TypeId<"edge">, sourceNodeId, targetNodeId, edgeType, description: string | null } @@ -121,6 +134,7 @@ Update an edge's type, description, or endpoints. ``` **Behavior:** + - Validates edge ownership; 404 if not found - If new node IDs provided, validates they exist and belong to userId; 404 if not - Updates edge row @@ -132,6 +146,7 @@ Update an edge's type, description, or endpoints. Merge multiple nodes into one. **Request:** + ```ts { userId: string @@ -142,6 +157,7 @@ Merge multiple nodes into one. ``` **Response:** + ```ts { node: { id: TypeId<"node">, nodeType: NodeType, label: string, description: string | null } @@ -149,6 +165,7 @@ Merge multiple nodes into one. ``` **Behavior:** + - Validates all nodes belong to userId - First node in array is the survivor - Accepts optional label/description override; otherwise keeps survivor's existing values @@ -166,19 +183,22 @@ Merge multiple nodes into one. Return node IDs associated with a given assistant's atlas. **Request:** + ```ts { - userId: string - assistantId: string + userId: string; + assistantId: string; } ``` **Response:** + ```ts { nodeIds: string[] } ``` **Behavior:** + - Finds the assistant's Atlas node (via `ensureAssistantAtlasNode` or equivalent) - Returns IDs of all nodes connected to it via edges @@ -187,6 +207,7 @@ Return node IDs associated with a given assistant's atlas. Add `nodeType: NodeTypeEnum.optional()` to `updateNodeRequestSchema`. **Behavior:** + - When provided, updates `nodes.nodeType` in addition to metadata fields - No other changes to the update flow @@ -195,19 +216,22 @@ Add `nodeType: NodeTypeEnum.optional()` to `updateNodeRequestSchema`. Atomic batch delete. **Request:** + ```ts { - userId: string - nodeIds: TypeId<"node">[] + userId: string; + nodeIds: TypeId < "node" > []; } ``` **Response:** + ```ts { deleted: true, count: number } ``` **Behavior:** + - Wraps all deletes in a single transaction - Returns count of actually deleted nodes @@ -218,6 +242,7 @@ Atomic batch delete. Add optional `nodeTypes: z.array(NodeTypeEnum).optional()` to `queryGraphRequestSchema`. **Behavior:** + - When provided, adds `WHERE nodes.nodeType IN (...)` to the graph query - Server-side filtering replaces client-side filtering of the 200-node cap @@ -226,6 +251,7 @@ Add optional `nodeTypes: z.array(NodeTypeEnum).optional()` to `queryGraphRequest Return the ego-graph around a focal node. **Request:** + ```ts { userId: string @@ -235,6 +261,7 @@ Return the ego-graph around a focal node. ``` **Response:** + ```ts { nodes: QueryGraphNode[], edges: QueryGraphEdge[] } ``` @@ -242,6 +269,7 @@ Return the ego-graph around a focal node. Same shape as `QueryGraphResponse` so the frontend needs no new type mappings. **Behavior:** + - Depth 1: focal node + all one-hop neighbors + edges between them - Depth 2: extends one more hop from the depth-1 set - Includes the focal node itself in the response @@ -249,6 +277,7 @@ Same shape as `QueryGraphResponse` so the frontend needs no new type mappings. ## New Files **Create:** + - `src/lib/schemas/edge.ts` - `src/lib/schemas/node-merge.ts` - `src/lib/schemas/node-batch-delete.ts` @@ -265,6 +294,7 @@ Same shape as `QueryGraphResponse` so the frontend needs no new type mappings. - `src/routes/query/atlas-nodes.ts` **Modify:** + - `src/lib/schemas/node.ts` — add create schema, add nodeType to update schema - `src/lib/schemas/query-graph.ts` — add nodeTypes filter - `src/lib/node.ts` — add createNode, mergeNodes, batchDeleteNodes, getNodeNeighborhood, update updateNode diff --git a/drizzle/0008_worthless_bullseye.sql b/drizzle/0008_worthless_bullseye.sql new file mode 100644 index 0000000..dc14bfa --- /dev/null +++ b/drizzle/0008_worthless_bullseye.sql @@ -0,0 +1,4 @@ +ALTER TABLE "node_metadata" ADD COLUMN "canonical_label" text;--> statement-breakpoint +-- Backfill canonical_label from existing labels +UPDATE "node_metadata" SET "canonical_label" = regexp_replace(lower(trim("label")), '\s+', ' ', 'g') WHERE "label" IS NOT NULL;--> statement-breakpoint +CREATE INDEX "node_metadata_canonical_label_idx" ON "node_metadata" USING btree ("canonical_label"); \ No newline at end of file diff --git a/drizzle/meta/0008_snapshot.json b/drizzle/meta/0008_snapshot.json new file mode 100644 index 0000000..291c32e --- /dev/null +++ b/drizzle/meta/0008_snapshot.json @@ -0,0 +1,1059 @@ +{ + "id": "ec92429f-cdba-4dc8-a78d-eb69e7ea7a46", + "prevId": "fd492a50-87a7-424d-8cc6-71e26d7aea55", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.aliases": { + "name": "aliases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "alias_text": { + "name": "alias_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_node_id": { + "name": "canonical_node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "aliases_user_id_users_id_fk": { + "name": "aliases_user_id_users_id_fk", + "tableFrom": "aliases", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "aliases_canonical_node_id_nodes_id_fk": { + "name": "aliases_canonical_node_id_nodes_id_fk", + "tableFrom": "aliases", + "tableTo": "nodes", + "columnsFrom": [ + "canonical_node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.edge_embeddings": { + "name": "edge_embeddings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "edge_id": { + "name": "edge_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1024)", + "primaryKey": false, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "edge_embeddings_embedding_idx": { + "name": "edge_embeddings_embedding_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "edge_embeddings_edge_id_idx": { + "name": "edge_embeddings_edge_id_idx", + "columns": [ + { + "expression": "edge_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "edge_embeddings_edge_id_edges_id_fk": { + "name": "edge_embeddings_edge_id_edges_id_fk", + "tableFrom": "edge_embeddings", + "tableTo": "edges", + "columnsFrom": [ + "edge_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.edges": { + "name": "edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_node_id": { + "name": "source_node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_node_id": { + "name": "target_node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "edge_type": { + "name": "edge_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "edges_user_id_source_node_id_idx": { + "name": "edges_user_id_source_node_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "edges_user_id_target_node_id_idx": { + "name": "edges_user_id_target_node_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "edges_user_id_edge_type_idx": { + "name": "edges_user_id_edge_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "edge_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "edges_user_id_users_id_fk": { + "name": "edges_user_id_users_id_fk", + "tableFrom": "edges", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "edges_source_node_id_nodes_id_fk": { + "name": "edges_source_node_id_nodes_id_fk", + "tableFrom": "edges", + "tableTo": "nodes", + "columnsFrom": [ + "source_node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "edges_target_node_id_nodes_id_fk": { + "name": "edges_target_node_id_nodes_id_fk", + "tableFrom": "edges", + "tableTo": "nodes", + "columnsFrom": [ + "target_node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "edges_sourceNodeId_targetNodeId_edge_type_unique": { + "name": "edges_sourceNodeId_targetNodeId_edge_type_unique", + "nullsNotDistinct": false, + "columns": [ + "source_node_id", + "target_node_id", + "edge_type" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.node_embeddings": { + "name": "node_embeddings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1024)", + "primaryKey": false, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "node_embeddings_embedding_idx": { + "name": "node_embeddings_embedding_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "node_embeddings_node_id_idx": { + "name": "node_embeddings_node_id_idx", + "columns": [ + { + "expression": "node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "node_embeddings_node_id_nodes_id_fk": { + "name": "node_embeddings_node_id_nodes_id_fk", + "tableFrom": "node_embeddings", + "tableTo": "nodes", + "columnsFrom": [ + "node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.node_metadata": { + "name": "node_metadata", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "canonical_label": { + "name": "canonical_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "additional_data": { + "name": "additional_data", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "node_metadata_node_id_idx": { + "name": "node_metadata_node_id_idx", + "columns": [ + { + "expression": "node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "node_metadata_canonical_label_idx": { + "name": "node_metadata_canonical_label_idx", + "columns": [ + { + "expression": "canonical_label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "node_metadata_node_id_nodes_id_fk": { + "name": "node_metadata_node_id_nodes_id_fk", + "tableFrom": "node_metadata", + "tableTo": "nodes", + "columnsFrom": [ + "node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "node_metadata_nodeId_unique": { + "name": "node_metadata_nodeId_unique", + "nullsNotDistinct": false, + "columns": [ + "node_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.nodes": { + "name": "nodes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "node_type": { + "name": "node_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "nodes_user_id_idx": { + "name": "nodes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "nodes_user_id_node_type_idx": { + "name": "nodes_user_id_node_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "node_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "nodes_user_id_users_id_fk": { + "name": "nodes_user_id_users_id_fk", + "tableFrom": "nodes", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scratchpads": { + "name": "scratchpads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scratchpads_user_id_idx": { + "name": "scratchpads_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scratchpads_user_id_users_id_fk": { + "name": "scratchpads_user_id_users_id_fk", + "tableFrom": "scratchpads", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "scratchpads_userId_unique": { + "name": "scratchpads_userId_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_links": { + "name": "source_links", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "specific_location": { + "name": "specific_location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_links_source_id_idx": { + "name": "source_links_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "source_links_node_id_idx": { + "name": "source_links_node_id_idx", + "columns": [ + { + "expression": "node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_links_source_id_sources_id_fk": { + "name": "source_links_source_id_sources_id_fk", + "tableFrom": "source_links", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "source_links_node_id_nodes_id_fk": { + "name": "source_links_node_id_nodes_id_fk", + "tableFrom": "source_links", + "tableTo": "nodes", + "columnsFrom": [ + "node_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_links_sourceId_nodeId_unique": { + "name": "source_links_sourceId_nodeId_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id", + "node_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_source": { + "name": "parent_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_ingested_at": { + "name": "last_ingested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_user_id_idx": { + "name": "sources_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_status_idx": { + "name": "sources_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_user_id_users_id_fk": { + "name": "sources_user_id_users_id_fk", + "tableFrom": "sources", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sources_userId_type_externalId_unique": { + "name": "sources_userId_type_externalId_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "type", + "external_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_profiles": { + "name": "user_profiles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_updated_at": { + "name": "last_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_profiles_user_id_users_id_fk": { + "name": "user_profiles_user_id_users_id_fk", + "tableFrom": "user_profiles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 080aa3f..05af970 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1772622104832, "tag": "0007_illegal_lila_cheney", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1775920239473, + "tag": "0008_worthless_bullseye", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/sync-obsidian.ts b/scripts/sync-obsidian.ts index 1c8f451..32987ca 100644 --- a/scripts/sync-obsidian.ts +++ b/scripts/sync-obsidian.ts @@ -20,9 +20,14 @@ * OBSIDIAN_MANIFEST - path to manifest file (default: scripts/.obsidian-sync-manifest.json) * OBSIDIAN_DRY_RUN - set to "true" to preview without ingesting */ - import { createHash } from "node:crypto"; -import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from "node:fs"; +import { + existsSync, + readFileSync, + writeFileSync, + readdirSync, + statSync, +} from "node:fs"; import { join, relative, extname } from "node:path"; // --------------------------------------------------------------------------- @@ -53,7 +58,10 @@ function loadConfig(): SyncConfig { const eqIdx = trimmed.indexOf("="); if (eqIdx === -1) continue; const key = trimmed.slice(0, eqIdx).trim(); - const val = trimmed.slice(eqIdx + 1).trim().replace(/^["']|["']$/g, ""); + const val = trimmed + .slice(eqIdx + 1) + .trim() + .replace(/^["']|["']$/g, ""); if (!process.env[key]) process.env[key] = val; } } @@ -64,7 +72,12 @@ function loadConfig(): SyncConfig { const userId = process.env["MEMORY_USER_ID"]; const parseList = (val: string | undefined): string[] => - val ? val.split(",").map((s) => s.trim()).filter(Boolean) : []; + val + ? val + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : []; return { vaultPath, @@ -111,7 +124,11 @@ function saveManifest(path: string, manifest: Manifest): void { // Vault reading // --------------------------------------------------------------------------- -function collectMarkdownFiles(dir: string, rootDir: string, config: SyncConfig): string[] { +function collectMarkdownFiles( + dir: string, + rootDir: string, + config: SyncConfig, +): string[] { const results: string[] = []; for (const entry of readdirSync(dir)) { @@ -187,7 +204,9 @@ function processFrontmatter(content: string): string { for (const line of frontmatter.split("\n")) { const tagMatch = line.match(/^tags:\s*\[(.+)\]$/); if (tagMatch) { - tags.push(...tagMatch[1]!.split(",").map((t) => t.trim().replace(/^#/, ""))); + tags.push( + ...tagMatch[1]!.split(",").map((t) => t.trim().replace(/^#/, "")), + ); } // Also handle YAML list format for tags const tagListMatch = line.match(/^\s*-\s*(.+)$/); @@ -202,7 +221,8 @@ function processFrontmatter(content: string): string { const metadataLines: string[] = []; if (tags.length > 0) metadataLines.push(`Tags: ${tags.join(", ")}`); - if (aliases.length > 0) metadataLines.push(`Also known as: ${aliases.join(", ")}`); + if (aliases.length > 0) + metadataLines.push(`Also known as: ${aliases.join(", ")}`); if (metadataLines.length > 0) { return `${metadataLines.join("\n")}\n\n${body}`; @@ -233,7 +253,9 @@ async function ingestDocument( config: SyncConfig, payload: IngestDocumentPayload, ): Promise<{ message: string; jobId: string }> { - const headers: Record = { "Content-Type": "application/json" }; + const headers: Record = { + "Content-Type": "application/json", + }; if (config.apiKey) headers["x-api-key"] = config.apiKey; const res = await fetch(`${config.apiUrl}/ingest/document`, { @@ -308,7 +330,11 @@ async function main() { console.log(); // Collect all markdown files - const files = collectMarkdownFiles(config.vaultPath, config.vaultPath, config); + const files = collectMarkdownFiles( + config.vaultPath, + config.vaultPath, + config, + ); console.log(`Found ${files.length} markdown files in vault`); // Load manifest and detect changes @@ -351,7 +377,10 @@ async function main() { console.log(` skip (too short): ${change.relPath}`); skipped++; // Still update manifest so we don't re-check every run - manifest[change.relPath] = { hash: change.hash, lastSynced: new Date().toISOString() }; + manifest[change.relPath] = { + hash: change.hash, + lastSynced: new Date().toISOString(), + }; continue; } @@ -369,7 +398,9 @@ async function main() { const documentId = `obsidian:${change.relPath}`; if (config.dryRun) { - console.log(` [dry run] ${change.type}: ${change.relPath} (${wc} words)`); + console.log( + ` [dry run] ${change.type}: ${change.relPath} (${wc} words)`, + ); ingested++; continue; } @@ -385,7 +416,10 @@ async function main() { }, }); console.log(` ${change.type}: ${change.relPath} → ${result.jobId}`); - manifest[change.relPath] = { hash: change.hash, lastSynced: new Date().toISOString() }; + manifest[change.relPath] = { + hash: change.hash, + lastSynced: new Date().toISOString(), + }; ingested++; } catch (err) { console.error(` ERROR: ${change.relPath} — ${err}`); @@ -409,9 +443,7 @@ async function main() { console.log( "Note: deleted files are removed from the manifest but their graph nodes are preserved.", ); - console.log( - "Use the API to manually delete document nodes if needed.", - ); + console.log("Use the API to manually delete document nodes if needed."); } // Save manifest diff --git a/src/db/schema.ts b/src/db/schema.ts index fc9cc46..1b19ada 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -56,18 +56,15 @@ export const nodeMetadata = pgTable( nodeId: typeId("node") .references(() => nodes.id, { onDelete: "cascade" }) .notNull(), - label: text(), // Human-readable name/title - description: text(), // Longer text description - // Maybe add timestamps for when this metadata was last updated - // Temporal aspect - can be added here or via edges - // validFrom: timestamp('valid_from'), - // validTo: timestamp('valid_to'), // Null means currently valid - additionalData: jsonb(), // For type-specific structured data + label: text(), + canonicalLabel: text("canonical_label"), + description: text(), + additionalData: jsonb(), createdAt: timestamp("created_at").defaultNow().notNull(), - // Index on nodeId }, (table) => [ index("node_metadata_node_id_idx").on(table.nodeId), + index("node_metadata_canonical_label_idx").on(table.canonicalLabel), unique().on(table.nodeId), ], ); diff --git a/src/lib/edge.ts b/src/lib/edge.ts index 5821843..7e45975 100644 --- a/src/lib/edge.ts +++ b/src/lib/edge.ts @@ -1,5 +1,4 @@ /** Edge operations: create, delete, update. */ - import { and, eq, inArray } from "drizzle-orm"; import { nodes, nodeMetadata, edges, edgeEmbeddings } from "~/db/schema"; import { generateEmbeddings } from "~/lib/embeddings"; @@ -57,7 +56,9 @@ export async function createEdge( }> { const db = await useDatabase(); - if (!(await validateNodeOwnership(db, userId, [sourceNodeId, targetNodeId]))) { + if ( + !(await validateNodeOwnership(db, userId, [sourceNodeId, targetNodeId])) + ) { throw new Error("One or both nodes not found"); } @@ -165,7 +166,10 @@ export async function updateEdge( const newSourceNodeId = updates.sourceNodeId ?? current.sourceNodeId; const newTargetNodeId = updates.targetNodeId ?? current.targetNodeId; const newEdgeType = updates.edgeType ?? current.edgeType; - const newDescription = updates.description !== undefined ? updates.description : current.description; + const newDescription = + updates.description !== undefined + ? updates.description + : current.description; const updateSet: Partial<{ edgeType: EdgeType; @@ -174,9 +178,15 @@ export async function updateEdge( targetNodeId: TypeId<"node">; }> = { ...(updates.edgeType !== undefined && { edgeType: updates.edgeType }), - ...(updates.description !== undefined && { description: updates.description }), - ...(updates.sourceNodeId !== undefined && { sourceNodeId: updates.sourceNodeId }), - ...(updates.targetNodeId !== undefined && { targetNodeId: updates.targetNodeId }), + ...(updates.description !== undefined && { + description: updates.description, + }), + ...(updates.sourceNodeId !== undefined && { + sourceNodeId: updates.sourceNodeId, + }), + ...(updates.targetNodeId !== undefined && { + targetNodeId: updates.targetNodeId, + }), }; if (Object.keys(updateSet).length > 0) { diff --git a/src/lib/extract-graph.ts b/src/lib/extract-graph.ts index 72ccf61..8ceb975 100644 --- a/src/lib/extract-graph.ts +++ b/src/lib/extract-graph.ts @@ -4,8 +4,10 @@ import { generateAndInsertEdgeEmbeddings, } from "./embeddings-util"; import { formatNodesForPrompt } from "./formatting"; -import { findSimilarNodes, findOneHopNodes } from "./graph"; +import { findSimilarNodes, findOneHopNodes, findNodesByType } from "./graph"; +import { normalizeLabel } from "./label"; import { TemporaryIdMapper } from "./temporary-id-mapper"; +import { and, eq, inArray } from "drizzle-orm"; import { zodResponseFormat } from "openai/helpers/zod.mjs"; import { z } from "zod"; import { type DrizzleDB } from "~/db"; @@ -73,8 +75,8 @@ export async function extractGraph({ }: ExtractGraphParams) { const db = await useDatabase(); - const similarNodesRaw = ( - await Promise.all([ + const [embeddingSimilar, oneHopNeighbors, allPersonNodes] = await Promise.all( + [ findSimilarNodes({ userId, text: content, @@ -82,19 +84,36 @@ export async function extractGraph({ minimumSimilarity: 0.3, }), findOneHopNodes(db, userId, [linkedNodeId]), - ]) - ).flat(); + findNodesByType(userId, "Person"), + ], + ); - const similarNodesForProcessing = similarNodesRaw.map((node) => ({ - id: node.id, - type: node.type, - label: node.label, - description: node.description, - timestamp: node.timestamp.toISOString(), - })); + // Deduplicate by node ID: person nodes first (most duplicated type), + // then embedding results, then one-hop neighbors + const seenIds = new Set>(); + const similarNodesForProcessing: SimilarNodeForPrompt[] = []; + + for (const node of [ + ...allPersonNodes, + ...embeddingSimilar, + ...oneHopNeighbors, + ]) { + if (seenIds.has(node.id)) continue; + seenIds.add(node.id); + similarNodesForProcessing.push({ + id: node.id, + type: node.type, + label: node.label, + description: node.description, + timestamp: node.timestamp.toISOString(), + }); + } + + // Cap to keep the prompt manageable + const cappedNodes = similarNodesForProcessing.slice(0, 150); const { nodesForPromptFormatting, idMap, nodeLabels } = - _prepareInitialNodeMappings(similarNodesForProcessing); + _prepareInitialNodeMappings(cappedNodes); const { createCompletionClient } = await import("./ai"); const client = await createCompletionClient(userId); @@ -313,11 +332,63 @@ async function _processAndInsertNewNodes( ): Promise { const detailsOfNewlyCreatedNodes: ProcessedNode[] = []; + // Batch dedup: collect all canonical labels for nodes not already in idMap, + // then look them all up in one query instead of N queries in the loop. + const newLlmNodes = uniqueParsedLlmNodes.filter((n) => !idMap.has(n.id)); + const canonicalLabels = newLlmNodes.map((n) => normalizeLabel(n.label)); + const uniqueCanonicals = [...new Set(canonicalLabels)].filter( + (c) => c !== "", + ); + + // Single batch query for all potential matches + const existingMatches = + uniqueCanonicals.length > 0 + ? await db + .select({ + id: nodes.id, + nodeType: nodes.nodeType, + label: nodeMetadata.label, + canonicalLabel: nodeMetadata.canonicalLabel, + }) + .from(nodes) + .innerJoin(nodeMetadata, eq(nodeMetadata.nodeId, nodes.id)) + .where( + and( + eq(nodes.userId, userId), + inArray(nodeMetadata.canonicalLabel, uniqueCanonicals), + ), + ) + : []; + + // Index by (nodeType, canonicalLabel) for O(1) lookup + const existingByKey = new Map< + string, + { id: TypeId<"node">; label: string | null } + >(); + for (const match of existingMatches) { + const key = `${match.nodeType}|${match.canonicalLabel}`; + if (!existingByKey.has(key)) { + existingByKey.set(key, { id: match.id, label: match.label }); + } + } + for (const llmNode of uniqueParsedLlmNodes) { if (idMap.has(llmNode.id)) { continue; } + // Exact-match dedup: check batch results for existing node + const canonical = normalizeLabel(llmNode.label); + const existing = existingByKey.get(`${llmNode.type}|${canonical}`); + + if (existing) { + idMap.set(llmNode.id, existing.id); + if (existing.label) { + nodeLabels.set(existing.id, existing.label); + } + continue; + } + const [insertedNodeRecord] = await db .insert(nodes) .values({ @@ -334,6 +405,7 @@ async function _processAndInsertNewNodes( await db.insert(nodeMetadata).values({ nodeId: insertedNodeRecord.id, label: llmNode.label, + canonicalLabel: canonical, description: llmNode.description, additionalData: {}, }); @@ -341,6 +413,13 @@ async function _processAndInsertNewNodes( idMap.set(llmNode.id, insertedNodeRecord.id); nodeLabels.set(insertedNodeRecord.id, llmNode.label); + // Also add to batch lookup so subsequent LLM nodes with the same label + // won't try to insert again + existingByKey.set(`${llmNode.type}|${canonical}`, { + id: insertedNodeRecord.id, + label: llmNode.label, + }); + detailsOfNewlyCreatedNodes.push({ id: insertedNodeRecord.id, label: llmNode.label, diff --git a/src/lib/graph.ts b/src/lib/graph.ts index c2e1ab5..c44ddb0 100644 --- a/src/lib/graph.ts +++ b/src/lib/graph.ts @@ -276,6 +276,35 @@ export async function findOneHopNodes( .limit(50); } +/** Fetch all nodes of a given type for a user */ +export async function findNodesByType( + userId: string, + nodeType: NodeType, + limit = 200, +): Promise { + const db = await useDatabase(); + return db + .select({ + id: nodes.id, + type: nodes.nodeType, + label: nodeMetadata.label, + description: nodeMetadata.description, + timestamp: nodes.createdAt, + similarity: sql`1`.as("similarity"), + }) + .from(nodes) + .innerJoin(nodeMetadata, eq(nodes.id, nodeMetadata.nodeId)) + .where( + and( + eq(nodes.userId, userId), + eq(nodes.nodeType, nodeType), + isNotNull(nodeMetadata.label), + ), + ) + .orderBy(desc(nodes.createdAt)) + .limit(limit); +} + /** Helper to fetch the Temporal day node id for a given userId and date */ export async function findDayNode( db: DrizzleDB, diff --git a/src/lib/jobs/atlas-improvements.test.ts b/src/lib/jobs/atlas-improvements.test.ts index 09fb9c6..17d7986 100644 --- a/src/lib/jobs/atlas-improvements.test.ts +++ b/src/lib/jobs/atlas-improvements.test.ts @@ -18,9 +18,7 @@ describe("Atlas Improvements", () => { expect(userAtlasContent).toContain( "Never include assistant speculation or assumptions", ); - expect(userAtlasContent).toContain( - "Include specific dates (YYYY-MM-DD)", - ); + expect(userAtlasContent).toContain("Include specific dates (YYYY-MM-DD)"); expect(userAtlasContent).toContain( "Update immediately if the user corrects or contradicts", ); diff --git a/src/lib/jobs/cleanup-graph.ts b/src/lib/jobs/cleanup-graph.ts index ee4759f..196198d 100644 --- a/src/lib/jobs/cleanup-graph.ts +++ b/src/lib/jobs/cleanup-graph.ts @@ -5,6 +5,7 @@ import { type EmbeddableEdge, } from "../embeddings-util"; import { findOneHopNodes, findSimilarNodes } from "../graph"; +import { normalizeLabel } from "../label"; import { TemporaryIdMapper } from "../temporary-id-mapper"; import { sql, eq, gte, desc, and, inArray } from "drizzle-orm"; import { zodResponseFormat } from "openai/helpers/zod.mjs"; @@ -502,9 +503,12 @@ async function applyCleanupProposal( .returning({ id: nodes.id }); const nodeId = inserted[0]?.id; if (!nodeId) continue; - await tx - .insert(nodeMetadata) - .values({ nodeId, label: n.label, description: n.description }); + await tx.insert(nodeMetadata).values({ + nodeId, + label: n.label, + canonicalLabel: normalizeLabel(n.label), + description: n.description, + }); createdNodes.push({ nodeId, label: n.label, @@ -687,7 +691,7 @@ async function applyCleanupProposal( /** * Rewire edges from removeId to keepId for a given user */ -async function rewireNodeEdges( +export async function rewireNodeEdges( tx: DrizzleDB, removeId: TypeId<"node">, keepId: TypeId<"node">, @@ -742,7 +746,7 @@ async function rewireNodeEdges( /** * Rewire source_links entries from removeId to keepId */ -async function rewireSourceLinks( +export async function rewireSourceLinks( tx: DrizzleDB, removeId: TypeId<"node">, keepId: TypeId<"node">, @@ -765,7 +769,7 @@ async function rewireSourceLinks( /** * Delete a node for a given user; cascades remove related data */ -async function deleteNode( +export async function deleteNode( tx: DrizzleDB, nodeId: TypeId<"node">, userId: string, diff --git a/src/lib/jobs/dedup-sweep.ts b/src/lib/jobs/dedup-sweep.ts new file mode 100644 index 0000000..9f75266 --- /dev/null +++ b/src/lib/jobs/dedup-sweep.ts @@ -0,0 +1,112 @@ +/** Deterministic dedup sweep: finds exact-label duplicate nodes and merges them. */ +import { + rewireNodeEdges, + rewireSourceLinks, + deleteNode, +} from "./cleanup-graph"; +import { and, eq, sql, isNotNull } from "drizzle-orm"; +import { DrizzleDB } from "~/db"; +import { nodes, nodeMetadata } from "~/db/schema"; +import { TypeId } from "~/types/typeid"; +import { useDatabase } from "~/utils/db"; + +interface DuplicateGroup { + nodeType: string; + canonicalLabel: string; + nodeIds: TypeId<"node">[]; +} + +/** + * Find all groups of nodes that share the same (userId, nodeType, canonicalLabel). + * Returns groups with 2+ members — these are duplicates. + */ +export async function findDuplicateGroups( + db: DrizzleDB, + userId: string, +): Promise { + const rows = await db + .select({ + nodeType: nodes.nodeType, + canonicalLabel: nodeMetadata.canonicalLabel, + nodeIds: sql< + TypeId<"node">[] + >`array_agg(${nodes.id} ORDER BY ${nodes.createdAt} ASC)`.as("node_ids"), + }) + .from(nodes) + .innerJoin(nodeMetadata, eq(nodeMetadata.nodeId, nodes.id)) + .where( + and( + eq(nodes.userId, userId), + isNotNull(nodeMetadata.canonicalLabel), + sql`trim(${nodeMetadata.canonicalLabel}) != ''`, + ), + ) + .groupBy(nodes.nodeType, nodeMetadata.canonicalLabel) + .having(sql`count(*) > 1`); + + return rows.map((r) => ({ + nodeType: r.nodeType, + canonicalLabel: r.canonicalLabel!, + nodeIds: r.nodeIds, + })); +} + +/** + * Merge a group of duplicate nodes: keep the oldest (first), rewire and delete the rest. + */ +async function mergeGroup( + tx: DrizzleDB, + userId: string, + group: DuplicateGroup, +): Promise { + const [keepId, ...removeIds] = group.nodeIds; + if (!keepId || removeIds.length === 0) return 0; + + for (const removeId of removeIds) { + await rewireNodeEdges(tx, removeId, keepId, userId); + await rewireSourceLinks(tx, removeId, keepId); + await deleteNode(tx, removeId, userId); + } + + return removeIds.length; +} + +export interface DedupSweepResult { + mergedGroups: number; + mergedNodes: number; +} + +/** + * Run a full dedup sweep for a user: find all exact-label duplicates and merge them. + */ +export async function runDedupSweep(userId: string): Promise { + const db = await useDatabase(); + const groups = await findDuplicateGroups(db, userId); + + if (groups.length === 0) { + console.log(`[dedup-sweep] No duplicates found for user ${userId}`); + return { mergedGroups: 0, mergedNodes: 0 }; + } + + console.log( + `[dedup-sweep] Found ${groups.length} duplicate groups for user ${userId}`, + ); + + let totalMerged = 0; + + await db.transaction(async (tx) => { + for (const group of groups) { + const merged = await mergeGroup(tx, userId, group); + totalMerged += merged; + console.log( + `[dedup-sweep] Merged ${merged} duplicates of "${group.canonicalLabel}" (${group.nodeType})`, + ); + } + }); + + console.log( + `[dedup-sweep] Completed: ${groups.length} groups, ${totalMerged} nodes merged for user ${userId}`, + ); + + return { mergedGroups: groups.length, mergedNodes: totalMerged }; +} diff --git a/src/lib/label.test.ts b/src/lib/label.test.ts new file mode 100644 index 0000000..ec64778 --- /dev/null +++ b/src/lib/label.test.ts @@ -0,0 +1,28 @@ +import { normalizeLabel } from "./label"; +import { describe, it, expect } from "vitest"; + +describe("normalizeLabel", () => { + it("lowercases and trims", () => { + expect(normalizeLabel(" Marcel Samyn ")).toBe("marcel samyn"); + }); + + it("collapses multiple spaces", () => { + expect(normalizeLabel("John Doe")).toBe("john doe"); + }); + + it("handles empty string", () => { + expect(normalizeLabel("")).toBe(""); + }); + + it("handles single word", () => { + expect(normalizeLabel("Alice")).toBe("alice"); + }); + + it("preserves non-ASCII characters", () => { + expect(normalizeLabel(" José García ")).toBe("josé garcía"); + }); + + it("handles tabs and newlines", () => { + expect(normalizeLabel("hello\t\nworld")).toBe("hello world"); + }); +}); diff --git a/src/lib/label.ts b/src/lib/label.ts new file mode 100644 index 0000000..dafa2b7 --- /dev/null +++ b/src/lib/label.ts @@ -0,0 +1,4 @@ +/** Normalize a node label for dedup comparison: lowercase, trim, collapse whitespace. */ +export function normalizeLabel(label: string): string { + return label.trim().replace(/\s+/g, " ").toLowerCase(); +} diff --git a/src/lib/mcp/mcp-server.ts b/src/lib/mcp/mcp-server.ts index 0765130..0574b49 100644 --- a/src/lib/mcp/mcp-server.ts +++ b/src/lib/mcp/mcp-server.ts @@ -5,12 +5,24 @@ import { } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { saveMemory } from "~/lib/ingestion/save-document"; +import { + getNodeById, + getNodeSources, + updateNode, + deleteNode, +} from "~/lib/node"; import { queryDayMemories } from "~/lib/query/day"; import { searchMemory } from "~/lib/query/search"; import { ingestDocumentRequestSchema, type IngestDocumentRequest, } from "~/lib/schemas/ingest-document-request"; +import { + getNodeRequestSchema, + getNodeSourcesRequestSchema, + updateNodeRequestSchema, + deleteNodeRequestSchema, +} from "~/lib/schemas/node"; import { queryDayRequestSchema, type QueryDayRequest, @@ -19,23 +31,11 @@ import { querySearchRequestSchema, type QuerySearchRequest, } from "~/lib/schemas/query-search"; -import { - getNodeRequestSchema, - getNodeSourcesRequestSchema, - updateNodeRequestSchema, - deleteNodeRequestSchema, -} from "~/lib/schemas/node"; import { scratchpadReadRequestSchema, scratchpadWriteRequestSchema, scratchpadEditRequestSchema, } from "~/lib/schemas/scratchpad"; -import { - getNodeById, - getNodeSources, - updateNode, - deleteNode, -} from "~/lib/node"; import { readScratchpad, writeScratchpad, @@ -201,7 +201,9 @@ server.tool( }; } return { - content: [{ type: "text", text: `Node updated: ${JSON.stringify(result)}` }], + content: [ + { type: "text", text: `Node updated: ${JSON.stringify(result)}` }, + ], }; }, ); diff --git a/src/lib/node.ts b/src/lib/node.ts index 1db5c70..f39946d 100644 --- a/src/lib/node.ts +++ b/src/lib/node.ts @@ -1,5 +1,5 @@ /** Node operations: get, get sources, update, delete. */ - +import type { GetNodeResponse, GetNodeSourcesResponse } from "./schemas/node"; import { and, eq, or, inArray, aliasedTable, sql } from "drizzle-orm"; import { nodes, @@ -10,11 +10,15 @@ import { sources, } from "~/db/schema"; import { generateEmbeddings } from "~/lib/embeddings"; -import { fetchSourceIdsForNodes, findOneHopNodes, fetchEdgesBetweenNodeIds } from "~/lib/graph"; +import { + fetchSourceIdsForNodes, + findOneHopNodes, + fetchEdgesBetweenNodeIds, +} from "~/lib/graph"; import { ensureUser } from "~/lib/ingestion/ensure-user"; -import type { NodeType } from "~/types/graph"; +import { normalizeLabel } from "~/lib/label"; import { sourceService } from "~/lib/sources"; -import type { GetNodeResponse, GetNodeSourcesResponse } from "./schemas/node"; +import type { NodeType } from "~/types/graph"; import type { TypeId } from "~/types/typeid"; import { useDatabase } from "~/utils/db"; @@ -108,9 +112,7 @@ export async function getNodeSources( if (linkedSources.length === 0) return { sources: [] }; // Fetch raw content for each source - const sourceIds = linkedSources.map( - (s) => s.sourceId as TypeId<"source">, - ); + const sourceIds = linkedSources.map((s) => s.sourceId as TypeId<"source">); const rawResults = await sourceService.fetchRaw(userId, sourceIds); const contentMap = new Map( rawResults.map((r) => [ @@ -134,7 +136,12 @@ export async function updateNode( userId: string, nodeId: TypeId<"node">, updates: { label?: string; description?: string; nodeType?: NodeType }, -): Promise<{ id: TypeId<"node">; nodeType: string; label: string | null; description: string | null } | null> { +): Promise<{ + id: TypeId<"node">; + nodeType: string; + label: string | null; + description: string | null; +} | null> { const db = await useDatabase(); // Verify ownership and fetch current state @@ -168,7 +175,12 @@ export async function updateNode( await db .update(nodeMetadata) .set({ - ...(updates.label !== undefined ? { label: updates.label } : {}), + ...(updates.label !== undefined + ? { + label: updates.label, + canonicalLabel: normalizeLabel(updates.label), + } + : {}), ...(updates.description !== undefined ? { description: updates.description } : {}), @@ -176,7 +188,10 @@ export async function updateNode( .where(eq(nodeMetadata.id, row.metaId)); // Re-generate embedding if label or description changed - if (newLabel && (updates.label !== undefined || updates.description !== undefined)) { + if ( + newLabel && + (updates.label !== undefined || updates.description !== undefined) + ) { const embText = `${newLabel}: ${newDescription ?? ""}`; const embResponse = await generateEmbeddings({ model: "jina-embeddings-v3", @@ -187,9 +202,7 @@ export async function updateNode( const embedding = embResponse.data[0]?.embedding; if (embedding) { // Delete old embedding and insert new one - await db - .delete(nodeEmbeddings) - .where(eq(nodeEmbeddings.nodeId, nodeId)); + await db.delete(nodeEmbeddings).where(eq(nodeEmbeddings.nodeId, nodeId)); await db.insert(nodeEmbeddings).values({ nodeId, embedding, @@ -227,7 +240,12 @@ export async function createNode( nodeType: NodeType, label: string, description?: string, -): Promise<{ id: TypeId<"node">; nodeType: NodeType; label: string; description: string | null }> { +): Promise<{ + id: TypeId<"node">; + nodeType: NodeType; + label: string; + description: string | null; +}> { const db = await useDatabase(); await ensureUser(db, userId); @@ -241,6 +259,7 @@ export async function createNode( await db.insert(nodeMetadata).values({ nodeId: inserted.id, label, + canonicalLabel: normalizeLabel(label), description: description ?? null, }); @@ -268,7 +287,12 @@ export async function mergeNodes( userId: string, nodeIds: TypeId<"node">[], overrides?: { targetLabel?: string; targetDescription?: string }, -): Promise<{ id: TypeId<"node">; nodeType: string; label: string; description: string | null } | null> { +): Promise<{ + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; +} | null> { const db = await useDatabase(); const foundNodes = await db @@ -349,9 +373,7 @@ export async function mergeNodes( ) `); - await tx - .delete(sourceLinks) - .where(eq(sourceLinks.nodeId, consumedId)); + await tx.delete(sourceLinks).where(eq(sourceLinks.nodeId, consumedId)); } // Delete consumed nodes @@ -362,7 +384,11 @@ export async function mergeNodes( // Update survivor metadata await tx .update(nodeMetadata) - .set({ label: finalLabel, description: finalDescription }) + .set({ + label: finalLabel, + canonicalLabel: normalizeLabel(finalLabel), + description: finalDescription, + }) .where(eq(nodeMetadata.nodeId, survivorId)); // Delete self-referencing edges @@ -383,7 +409,9 @@ export async function mergeNodes( }); const embedding = embResponse.data[0]?.embedding; if (embedding) { - await db.delete(nodeEmbeddings).where(eq(nodeEmbeddings.nodeId, survivorId)); + await db + .delete(nodeEmbeddings) + .where(eq(nodeEmbeddings.nodeId, survivorId)); await db.insert(nodeEmbeddings).values({ nodeId: survivorId, embedding, @@ -418,8 +446,19 @@ export async function getNodeNeighborhood( nodeId: TypeId<"node">, depth: 1 | 2 = 1, ): Promise<{ - nodes: { id: TypeId<"node">; nodeType: string; label: string; description: string | null; sourceIds: string[] }[]; - edges: { source: TypeId<"node">; target: TypeId<"node">; edgeType: string; description: string | null }[]; + nodes: { + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; + sourceIds: string[]; + }[]; + edges: { + source: TypeId<"node">; + target: TypeId<"node">; + edgeType: string; + description: string | null; + }[]; } | null> { const db = await useDatabase(); @@ -440,7 +479,12 @@ export async function getNodeNeighborhood( const allNodeIds = new Set>([nodeId]); const nodeMap = new Map< TypeId<"node">, - { id: TypeId<"node">; nodeType: string; label: string; description: string | null } + { + id: TypeId<"node">; + nodeType: string; + label: string; + description: string | null; + } >(); nodeMap.set(nodeId, { id: focal.id, diff --git a/src/lib/queues.ts b/src/lib/queues.ts index 5bb3367..a8b1d9c 100644 --- a/src/lib/queues.ts +++ b/src/lib/queues.ts @@ -101,6 +101,10 @@ const worker = new Worker( `Ingested conversation ${conversationId} for user ${userId}.`, ); + // Run dedup sweep after ingestion to clean up any duplicates + const { runDedupSweep } = await import("./jobs/dedup-sweep"); + await runDedupSweep(userId); + // Queue deep research job if there are messages if (messages.length > 0) { // Simple throttling: add a low probability to reduce job frequency @@ -170,6 +174,12 @@ const worker = new Worker( updateExisting, }); console.log(`Ingested document ${documentId} for user ${userId}.`); + + // Run dedup sweep after ingestion + const { runDedupSweep: runDocDedupSweep } = await import( + "./jobs/dedup-sweep" + ); + await runDocDedupSweep(userId); } else if (job.name === "cleanup-graph") { const data = CleanupGraphJobInputSchema.parse({ ...job.data, @@ -193,6 +203,12 @@ const worker = new Worker( `Basic cleanup completed: truncated ${truncateResult.updatedCount} labels, generated ${embeddingsResult.generatedCount} embeddings`, ); + // Run deterministic dedup sweep before LLM-based cleanup + const { runDedupSweep: runCleanupDedupSweep } = await import( + "./jobs/dedup-sweep" + ); + await runCleanupDedupSweep(data.userId); + // Then run the iterative graph cleanup const { runIterativeCleanup } = await import( "./jobs/run-iterative-cleanup" diff --git a/src/lib/schemas/cleanup.ts b/src/lib/schemas/cleanup.ts index 0d63785..4faf421 100644 --- a/src/lib/schemas/cleanup.ts +++ b/src/lib/schemas/cleanup.ts @@ -16,3 +16,15 @@ export const cleanupResponseSchema = z.object({ export type CleanupRequest = z.infer; export type CleanupResponse = z.infer; + +export const dedupSweepRequestSchema = z.object({ + userId: z.string().startsWith("user_"), +}); + +export const dedupSweepResponseSchema = z.object({ + mergedGroups: z.number(), + mergedNodes: z.number(), +}); + +export type DedupSweepRequest = z.infer; +export type DedupSweepResponse = z.infer; diff --git a/src/lib/schemas/node-batch-delete.ts b/src/lib/schemas/node-batch-delete.ts index 338a7d3..d8f922c 100644 --- a/src/lib/schemas/node-batch-delete.ts +++ b/src/lib/schemas/node-batch-delete.ts @@ -11,5 +11,9 @@ export const batchDeleteNodesResponseSchema = z.object({ count: z.number().int().nonnegative(), }); -export type BatchDeleteNodesRequest = z.infer; -export type BatchDeleteNodesResponse = z.infer; +export type BatchDeleteNodesRequest = z.infer< + typeof batchDeleteNodesRequestSchema +>; +export type BatchDeleteNodesResponse = z.infer< + typeof batchDeleteNodesResponseSchema +>; diff --git a/src/lib/schemas/node-neighborhood.ts b/src/lib/schemas/node-neighborhood.ts index 8b436ab..bfb18bc 100644 --- a/src/lib/schemas/node-neighborhood.ts +++ b/src/lib/schemas/node-neighborhood.ts @@ -13,5 +13,9 @@ export const nodeNeighborhoodResponseSchema = z.object({ edges: z.array(queryGraphEdgeSchema), }); -export type NodeNeighborhoodRequest = z.infer; -export type NodeNeighborhoodResponse = z.infer; +export type NodeNeighborhoodRequest = z.infer< + typeof nodeNeighborhoodRequestSchema +>; +export type NodeNeighborhoodResponse = z.infer< + typeof nodeNeighborhoodResponseSchema +>; diff --git a/src/lib/schemas/node.ts b/src/lib/schemas/node.ts index 658ea02..1d963a1 100644 --- a/src/lib/schemas/node.ts +++ b/src/lib/schemas/node.ts @@ -52,9 +52,7 @@ export const getNodeSourcesResponseSchema = z.object({ sources: z.array(nodeSourceSchema), }); -export type GetNodeSourcesRequest = z.infer< - typeof getNodeSourcesRequestSchema ->; +export type GetNodeSourcesRequest = z.infer; export type GetNodeSourcesResponse = z.infer< typeof getNodeSourcesResponseSchema >; diff --git a/src/lib/schemas/query-atlas-nodes.ts b/src/lib/schemas/query-atlas-nodes.ts index 2959235..4d92dfe 100644 --- a/src/lib/schemas/query-atlas-nodes.ts +++ b/src/lib/schemas/query-atlas-nodes.ts @@ -9,5 +9,9 @@ export const queryAtlasNodesResponseSchema = z.object({ nodeIds: z.array(z.string()), }); -export type QueryAtlasNodesRequest = z.infer; -export type QueryAtlasNodesResponse = z.infer; +export type QueryAtlasNodesRequest = z.infer< + typeof queryAtlasNodesRequestSchema +>; +export type QueryAtlasNodesResponse = z.infer< + typeof queryAtlasNodesResponseSchema +>; diff --git a/src/routes/cleanup/dedup-sweep.post.ts b/src/routes/cleanup/dedup-sweep.post.ts new file mode 100644 index 0000000..2471f8c --- /dev/null +++ b/src/routes/cleanup/dedup-sweep.post.ts @@ -0,0 +1,12 @@ +import { defineEventHandler, readBody } from "h3"; +import { runDedupSweep } from "~/lib/jobs/dedup-sweep"; +import { + dedupSweepRequestSchema, + dedupSweepResponseSchema, +} from "~/lib/schemas/cleanup"; + +export default defineEventHandler(async (event) => { + const { userId } = dedupSweepRequestSchema.parse(await readBody(event)); + const result = await runDedupSweep(userId); + return dedupSweepResponseSchema.parse(result); +}); diff --git a/src/routes/node/get.post.ts b/src/routes/node/get.post.ts index fa96a5b..0afa23b 100644 --- a/src/routes/node/get.post.ts +++ b/src/routes/node/get.post.ts @@ -6,9 +6,7 @@ import { } from "~/lib/schemas/node"; export default defineEventHandler(async (event) => { - const { userId, nodeId } = getNodeRequestSchema.parse( - await readBody(event), - ); + const { userId, nodeId } = getNodeRequestSchema.parse(await readBody(event)); const result = await getNodeById(userId, nodeId); if (!result) { throw createError({ statusCode: 404, statusMessage: "Node not found" }); diff --git a/src/routes/node/update.post.ts b/src/routes/node/update.post.ts index 1d538dc..9642efc 100644 --- a/src/routes/node/update.post.ts +++ b/src/routes/node/update.post.ts @@ -8,7 +8,11 @@ import { export default defineEventHandler(async (event) => { const { userId, nodeId, label, description, nodeType } = updateNodeRequestSchema.parse(await readBody(event)); - const result = await updateNode(userId, nodeId, { label, description, nodeType }); + const result = await updateNode(userId, nodeId, { + label, + description, + nodeType, + }); if (!result) { throw createError({ statusCode: 404, statusMessage: "Node not found" }); } diff --git a/src/routes/query/atlas-nodes.ts b/src/routes/query/atlas-nodes.ts index f59250b..5935112 100644 --- a/src/routes/query/atlas-nodes.ts +++ b/src/routes/query/atlas-nodes.ts @@ -1,11 +1,11 @@ +import { and, eq, or } from "drizzle-orm"; import { defineEventHandler } from "h3"; +import { edges } from "~/db/schema"; import { ensureAssistantAtlasNode } from "~/lib/atlas"; import { queryAtlasNodesRequestSchema, queryAtlasNodesResponseSchema, } from "~/lib/schemas/query-atlas-nodes"; -import { and, eq, or } from "drizzle-orm"; -import { edges } from "~/db/schema"; import { useDatabase } from "~/utils/db"; export default defineEventHandler(async (event) => { diff --git a/src/sdk/memory-client.ts b/src/sdk/memory-client.ts index ab27257..bce60fa 100644 --- a/src/sdk/memory-client.ts +++ b/src/sdk/memory-client.ts @@ -8,23 +8,6 @@ import { DreamResponse, dreamResponseSchema, } from "../lib/schemas/dream.js"; -import { - GetNodeRequest, - GetNodeResponse, - getNodeResponseSchema, - GetNodeSourcesRequest, - GetNodeSourcesResponse, - getNodeSourcesResponseSchema, - UpdateNodeRequest, - UpdateNodeResponse, - updateNodeResponseSchema, - DeleteNodeRequest, - DeleteNodeResponse, - deleteNodeResponseSchema, - CreateNodeRequest, - CreateNodeResponse, - createNodeResponseSchema, -} from "../lib/schemas/node.js"; import { CreateEdgeRequest, CreateEdgeResponse, @@ -37,44 +20,52 @@ import { updateEdgeResponseSchema, } from "../lib/schemas/edge.js"; import { - MergeNodesRequest, - MergeNodesResponse, - mergeNodesResponseSchema, -} from "../lib/schemas/node-merge.js"; + IngestConversationRequest, + IngestConversationResponse, + ingestConversationResponseSchema, +} from "../lib/schemas/ingest-conversation.js"; +import { + IngestDocumentRequest, + IngestDocumentResponse, + ingestDocumentResponseSchema, +} from "../lib/schemas/ingest-document-request.js"; import { BatchDeleteNodesRequest, BatchDeleteNodesResponse, batchDeleteNodesResponseSchema, } from "../lib/schemas/node-batch-delete.js"; import { - QueryAtlasNodesRequest, - QueryAtlasNodesResponse, - queryAtlasNodesResponseSchema, -} from "../lib/schemas/query-atlas-nodes.js"; + MergeNodesRequest, + MergeNodesResponse, + mergeNodesResponseSchema, +} from "../lib/schemas/node-merge.js"; import { NodeNeighborhoodRequest, NodeNeighborhoodResponse, nodeNeighborhoodResponseSchema, } from "../lib/schemas/node-neighborhood.js"; import { - ScratchpadReadRequest, - ScratchpadWriteRequest, - ScratchpadEditRequest, - ScratchpadResponse, - ScratchpadEditResponse, - scratchpadResponseSchema, - scratchpadEditResponseSchema, -} from "../lib/schemas/scratchpad.js"; -import { - IngestConversationRequest, - IngestConversationResponse, - ingestConversationResponseSchema, -} from "../lib/schemas/ingest-conversation.js"; + GetNodeRequest, + GetNodeResponse, + getNodeResponseSchema, + GetNodeSourcesRequest, + GetNodeSourcesResponse, + getNodeSourcesResponseSchema, + UpdateNodeRequest, + UpdateNodeResponse, + updateNodeResponseSchema, + DeleteNodeRequest, + DeleteNodeResponse, + deleteNodeResponseSchema, + CreateNodeRequest, + CreateNodeResponse, + createNodeResponseSchema, +} from "../lib/schemas/node.js"; import { - IngestDocumentRequest, - IngestDocumentResponse, - ingestDocumentResponseSchema, -} from "../lib/schemas/ingest-document-request.js"; + QueryAtlasNodesRequest, + QueryAtlasNodesResponse, + queryAtlasNodesResponseSchema, +} from "../lib/schemas/query-atlas-nodes.js"; import { QueryAtlasRequest, QueryAtlasResponse, @@ -100,6 +91,15 @@ import { QuerySearchResponse, querySearchResponseSchema, } from "../lib/schemas/query-search.js"; +import { + ScratchpadReadRequest, + ScratchpadWriteRequest, + ScratchpadEditRequest, + ScratchpadResponse, + ScratchpadEditResponse, + scratchpadResponseSchema, + scratchpadEditResponseSchema, +} from "../lib/schemas/scratchpad.js"; import { SummarizeRequest, SummarizeResponse,