diff --git a/.gitignore b/.gitignore index 215a1f8..9501b78 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ Thumbs.db .idea/ .vscode/ *.swp +.openvisio/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..7ac0ab7 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + "openvisio": { + "command": "openvisio", + "args": [ + "mcp", + ".", + "--watch" + ] + } + } +} diff --git a/.openvisio/cache/openvisio-oss/data.mdb b/.openvisio/cache/openvisio-oss/data.mdb index 5954964..ba0832d 100644 Binary files a/.openvisio/cache/openvisio-oss/data.mdb and b/.openvisio/cache/openvisio-oss/data.mdb differ diff --git a/.openvisio/cache/openvisio-oss/lock.mdb b/.openvisio/cache/openvisio-oss/lock.mdb index 70791e7..d946af5 100644 Binary files a/.openvisio/cache/openvisio-oss/lock.mdb and b/.openvisio/cache/openvisio-oss/lock.mdb differ diff --git a/README.md b/README.md index eb0426b..244ccd8 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,34 @@ viewer, and the CLI), then `node mcp/dist/cli.js view .`. --- +## Share a graph (transport) + +`view` keeps everything on `127.0.0.1`. When you want a **shareable link** — the +same Atlas/City map plus an AI **narrator** — without uploading your source, +`transport` does the split: it indexes the repo **locally** (the heavy +tree-sitter scan stays on your machine, private and fast) and ships **only the +computed graph JSON** to a web server that renders it. + +```bash +openvisio transport # index the current repo, upload the graph, open the link +openvisio transport ../other # …or any other local repo +openvisio transport --no-open # just print the URL, don't open a browser +openvisio transport --server=https://your-host # send to your own deployment +``` + +What happens: + +1. Index locally → build the deterministic graph. +2. Write a cached copy to `/.openvisio/graph.json` (auto-added to `.gitignore`). +3. `POST` just that graph JSON to `/api/import`. +4. Open the rendered graph + narrator at `/?g=`. + +Your source never leaves your machine — only the symbol/import graph is sent. The +destination defaults to `https://openvisio.io`; override it per-run with +`--server` or globally with the `OPENVISIO_SERVER` environment variable. + +--- + ## Languages OpenVisio parses these into symbols and import/call edges (tree-sitter grammars). diff --git a/core/.gitignore b/core/.gitignore new file mode 100644 index 0000000..30de003 --- /dev/null +++ b/core/.gitignore @@ -0,0 +1 @@ +.openvisio/ diff --git a/core/package.json b/core/package.json index 696e364..a36f07e 100644 --- a/core/package.json +++ b/core/package.json @@ -1,6 +1,6 @@ { "name": "@openvisio/core", - "version": "0.2.0", + "version": "0.2.1", "description": "Deterministic, local-first code-graph engine for OpenVisio: tree-sitter symbol/import extraction, PageRank ranking, token-budgeted skeletons and slices. No LLM, no network.", "type": "module", "license": "MIT", @@ -20,6 +20,10 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./browser": { + "types": "./dist/browser.d.ts", + "default": "./dist/browser.js" } }, "files": [ diff --git a/core/src/browser.ts b/core/src/browser.ts new file mode 100644 index 0000000..e79c43a --- /dev/null +++ b/core/src/browser.ts @@ -0,0 +1,15 @@ +// Browser-safe surface of @openvisio/core. The engine's parse layer is pure +// (tree-sitter queries + per-language resolution logic) — only the scanner, +// graph builder, and wasm *loader* touch Node I/O. This entry re-exports just +// the fs-free pieces, so a browser bundle (the in-browser GitHub indexer) reuses +// the EXACT grammar definitions + import resolution the CLI uses, with no Node +// dependencies dragged in. +// +// What it deliberately does NOT export: scan.ts, build.ts, treesitter.ts, +// store/* (all import node:fs / node:module). The browser provides its own +// in-memory scan + a fetch-based wasm loader instead. + +export { GRAMMARS, type GrammarConfig, type TsAliases } from './parse/grammars/index.js' +export { EXT_TO_GRAMMAR, grammarIdFromPath } from './parse/extensions.js' +export type { GrammarId } from './parse/treesitter.js' +export type { Language, SymbolKind, ParseResult } from './types.js' diff --git a/mcp/README.md b/mcp/README.md index 126f1fd..0c7ab63 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -120,6 +120,7 @@ every turn, so a fat surface re-creates the bloat we're removing. openvisio [init] [path] [--global] [--cursor] # register with agents + first index openvisio mcp [path] [--watch] [--spotlight] # MCP server over stdio openvisio view [path] [--port=7077] [--no-open] # open the local graph viewer in a browser +openvisio transport [path] [--server=https://openvisio.io] [--no-open] # index locally, ship the graph to a web viewer openvisio skeleton [path] [--budget=1500] [--task="add oauth"] # print the ranked map openvisio export [path] [--out=.openvisio/graph.json] # emit the graph as JSON ``` @@ -142,6 +143,13 @@ openvisio export [path] [--out=.openvisio/graph.json] # emit the graph spotlight port (7077) and acts as the highlight hub: with `view` running, start your agent with `openvisio mcp . --spotlight` and its tool calls focus the graph live. +- `transport` is the shareable cousin of `view`: it indexes the repo **locally** + (your source stays on your machine), writes a cached copy to + `/.openvisio/graph.json`, then `POST`s **only the computed graph JSON** to + `/api/import` and opens the rendered Atlas/City + AI narrator at + `/?g=`. The destination defaults to `https://openvisio.io`; override + it with `--server` or the `OPENVISIO_SERVER` env var. `--no-open` just prints the + URL. --- diff --git a/mcp/package.json b/mcp/package.json index 86bc669..ba48777 100644 --- a/mcp/package.json +++ b/mcp/package.json @@ -1,6 +1,6 @@ { "name": "openvisio", - "version": "0.2.0", + "version": "0.2.1", "description": "See any codebase as a graph. MCP server + CLI that serves coding agents (Claude Code, Codex, Cursor) a token-cheap, ranked, graph-native query surface over a local repo — agents query structure instead of crawling files. Local-first, read-only, no network.", "type": "module", "license": "MIT", diff --git a/mcp/src/cli.ts b/mcp/src/cli.ts index 580a5d1..104ffd2 100644 --- a/mcp/src/cli.ts +++ b/mcp/src/cli.ts @@ -12,6 +12,7 @@ import { toExportPayload } from './adapter.js' import { serveMcp } from './server.js' import { serveAgent } from './agent.js' import { serveViewer } from './viewer.js' +import { runTransport } from './transport.js' import { startSpotlightServer } from './spotlight.js' import { runInit, runGlobalInit } from './init.js' @@ -88,6 +89,12 @@ function printUsage(): void { ' Build the graph and write it where the viewer reads it (default:', ' /.openvisio/graph.json). Prints a one-line summary, not the JSON.', '', + ' openvisio transport [path] [--server=https://openvisio.io] [--no-open]', + ' Index the repo LOCALLY, then ship just the graph JSON to the web', + ' server and open the rendered graph + narrator in your browser. Your', + ' source never leaves your machine — only the computed graph is sent.', + ' Override the destination with --server or OPENVISIO_SERVER.', + '', ' openvisio view [path] [--port=7077] [--no-open]', ' Index a local repo and open the bundled graph viewer in your browser.', ' A self-contained UI (no install) that draws the same graph the MCP', @@ -316,6 +323,15 @@ async function main(): Promise { ) return 0 } + case 'transport': { + // Index locally, then ship the graph JSON to a web server that renders it. + const root = args.positional[0] ?? process.cwd() + const serverRaw = args.flags.get('server') + const server = serverRaw && serverRaw !== 'true' ? serverRaw : process.env.OPENVISIO_SERVER || 'https://openvisio.io' + const outRaw = args.flags.get('out') + const out = outRaw && outRaw !== 'true' ? outRaw : undefined + return runTransport({ rootPath: root, server, out, open: !args.flags.has('no-open') }) + } case 'help': case '--help': printUsage() diff --git a/mcp/src/transport.ts b/mcp/src/transport.ts new file mode 100644 index 0000000..f9fde81 --- /dev/null +++ b/mcp/src/transport.ts @@ -0,0 +1,115 @@ +// `openvisio transport [path]` — index a repo LOCALLY, then ship just the graph +// JSON to a web server that renders it. This is the local-repo half of the hosted +// viewer: the heavy work (clone-free filesystem scan + tree-sitter parse) happens +// on your machine where it's fast and private; the server only stores + renders a +// pre-computed graph (no git, no indexing, no source upload). +// +// Flow: build the graph → write .openvisio/graph.json → POST it to +// /api/import → open /?g= (the rendered graph + narrator). + +import { spawn } from 'node:child_process' +import * as fs from 'node:fs' +import * as path from 'node:path' +import { buildGraph } from '@openvisio/core' +import { toExportPayload } from './adapter.js' + +export interface TransportOptions { + /** Repo to index. */ + rootPath: string + /** Web server base URL, e.g. https://openvisio.io (no trailing slash). */ + server: string + /** Also write the graph to this file (default /.openvisio/graph.json). */ + out?: string + /** Open the system browser at the rendered URL (default true). */ + open: boolean +} + +/** Add `.openvisio/` to the repo's .gitignore once (best-effort). */ +function ensureGitIgnore(root: string): void { + const gitignorePath = path.join(root, '.gitignore') + const entry = '.openvisio/' + let content = '' + try { + content = fs.readFileSync(gitignorePath, 'utf8') + } catch { + /* no .gitignore — will create */ + } + if (content.split(/\r?\n/).some((l) => l.trim() === entry)) return + const sep = content.length === 0 || content.endsWith('\n') ? '' : '\n' + try { + fs.writeFileSync(gitignorePath, content + sep + entry + '\n') + } catch { + /* read-only fs — fine */ + } +} + +/** Open `url` in the system default browser (best-effort, never throws). */ +function openBrowser(url: string): void { + const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'cmd' : 'xdg-open' + const args = process.platform === 'win32' ? ['/c', 'start', '', url] : [url] + try { + const child = spawn(cmd, args, { stdio: 'ignore', detached: true }) + child.on('error', () => {}) + child.unref() + } catch { + /* headless box — the URL is printed for the user */ + } +} + +export async function runTransport(opts: TransportOptions): Promise { + const root = path.resolve(opts.rootPath) + const server = opts.server.replace(/\/+$/, '') + + // 1. Index locally. + process.stderr.write(`openvisio transport: indexing ${root} …\n`) + const started = process.hrtime.bigint() + const graph = await buildGraph(root) + const payload = toExportPayload(graph, Date.now()) + const json = JSON.stringify(payload) + const buildMs = Number(process.hrtime.bigint() - started) / 1e6 + process.stderr.write( + `openvisio transport: indexed ${graph.files.length} files, ${graph.symbols.length} symbols, ` + + `${graph.edges.length} edges in ${buildMs.toFixed(0)}ms (${(Buffer.byteLength(json) / 1024 / 1024).toFixed(1)} MB)\n`, + ) + + // 2. Write the graph to the local .openvisio folder (cache + offline copy). + const outFile = opts.out ?? path.join(root, '.openvisio', 'graph.json') + try { + fs.mkdirSync(path.dirname(outFile), { recursive: true }) + fs.writeFileSync(outFile, json) + ensureGitIgnore(root) + process.stderr.write(`openvisio transport: wrote ${outFile}\n`) + } catch (err) { + process.stderr.write(`openvisio transport: could not write ${outFile} (${err instanceof Error ? err.message : String(err)})\n`) + } + + // 3. Upload the graph JSON to the server. + process.stderr.write(`openvisio transport: uploading to ${server}/api/import …\n`) + let res: Response + try { + res = await fetch(`${server}/api/import`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Openvisio-Name': graph.name }, + body: json, + }) + } catch (err) { + process.stderr.write(`openvisio transport: upload failed — could not reach ${server} (${err instanceof Error ? err.message : String(err)})\n`) + return 1 + } + if (!res.ok) { + const detail = await res.text().catch(() => '') + process.stderr.write(`openvisio transport: upload rejected (${res.status}) ${detail.slice(0, 200)}\n`) + return 1 + } + const body = (await res.json().catch(() => ({}))) as { id?: string; url?: string } + if (!body.id && !body.url) { + process.stderr.write('openvisio transport: server did not return an id/url\n') + return 1 + } + + // 4/5. Open the rendered graph + narrator. + const viewUrl = body.url ?? `${server}/?g=${encodeURIComponent(body.id!)}` + process.stderr.write(`openvisio transport: live at\n ${viewUrl}\n`) + if (opts.open) openBrowser(viewUrl) + return 0 +} diff --git a/package-lock.json b/package-lock.json index d2fa0e0..e70bdf6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "openvisio", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openvisio", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "workspaces": [ "core", @@ -19,7 +19,7 @@ }, "core": { "name": "@openvisio/core", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "dependencies": { "lmdb": "^3.2.6", @@ -33,13 +33,13 @@ }, "mcp": { "name": "openvisio", - "version": "0.1.5", + "version": "0.2.0", "hasInstallScript": true, "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "lmdb": "^3.2.6", - "openvisio-viewer": "^0.1.0", + "openvisio-viewer": "^0.2.0", "tree-sitter-wasms": "^0.1.13", "web-tree-sitter": "~0.25.10", "zod": "^4.4.3" @@ -1139,6 +1139,16 @@ "win32" ] }, + "node_modules/@netlify/blobs": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-8.2.0.tgz", + "integrity": "sha512-9djLZHBKsoKk8XCgwWSEPK9QnT8qqxEQGuYh48gFIcNLvpBKkLnHbDZuyUxmNemCfDz7h0HnMXgSPnnUVgARhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, "node_modules/@openvisio/core": { "resolved": "core", "link": true @@ -4116,6 +4126,13 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true, + "license": "MIT" + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -5168,9 +5185,11 @@ }, "viewer": { "name": "openvisio-viewer", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "devDependencies": { + "@netlify/blobs": "^8.1.0", + "@openvisio/core": "*", "@react-three/drei": "^10.7.7", "@react-three/fiber": "^9.6.1", "@tailwindcss/vite": "^4.0.0", @@ -5180,6 +5199,7 @@ "@vitejs/plugin-react": "^4.3.4", "clsx": "^2.1.1", "lucide-react": "^0.460.0", + "path-browserify": "^1.0.1", "react": "^19.0.0", "react-dom": "^19.0.0", "tailwind-merge": "^2.5.5", @@ -5188,6 +5208,7 @@ "three-stdlib": "^2.35.0", "typescript": "^5.7.2", "vite": "^6.0.0", + "web-tree-sitter": "~0.25.10", "zod": "^3.23.8" } }, diff --git a/package.json b/package.json index 6ce21e1..406ac7f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openvisio", - "version": "0.2.0", + "version": "0.2.1", "private": true, "description": "OpenVisio monorepo. The deterministic code-graph core, the standalone MCP CLI published to npm as `openvisio`, and a local-first viewer (Atlas + City). See any codebase as a graph; give coding agents a token-cheap, graph-native query surface.", "license": "MIT", diff --git a/viewer/netlify.toml b/viewer/netlify.toml new file mode 100644 index 0000000..2051924 --- /dev/null +++ b/viewer/netlify.toml @@ -0,0 +1,21 @@ +# Netlify deploy for the OpenVisio viewer (static Vite SPA + the transport +# import function). Everything here runs on Netlify's free tier: +# • the viewer (static) — paste a GitHub URL → index in YOUR browser, no server +# • /api/import (function + Netlify Blobs) — receives graphs from +# `openvisio transport` and serves them back to /?g= +# No persistent server, no git, no source upload. + +[build] + command = "npm run build" + publish = "dist" + +[functions] + directory = "netlify/functions" + node_bundler = "esbuild" + +# SPA fallback: client routes (/?g=…) serve index.html. The /api/import function +# is routed by its own `config.path`, which takes precedence over this rewrite. +[[redirects]] + from = "/*" + to = "/index.html" + status = 200 diff --git a/viewer/netlify/functions/import.mts b/viewer/netlify/functions/import.mts new file mode 100644 index 0000000..2c00125 --- /dev/null +++ b/viewer/netlify/functions/import.mts @@ -0,0 +1,82 @@ +// Transport endpoint for the hosted viewer. `openvisio transport` POSTs a +// pre-computed graph JSON here; the viewer GETs it back by id via /?g=. The +// server stores + serves — it never indexes or sees source. Storage is Netlify +// Blobs (free tier), so this whole thing runs on Netlify's no-cost plan. + +import { getStore, type Store } from '@netlify/blobs' +import { randomUUID } from 'node:crypto' + +// Netlify Functions cap a synchronous request body at ~6 MB; reject larger +// graphs with a clear pointer (they should index a smaller subtree, or we add +// gzip later). Measured in BYTES — `string.length` counts UTF-16 code units, so +// a graph with multibyte content could slip past a length check and then get +// killed by the platform as an opaque 500. +const MAX_BYTES = 6 * 1024 * 1024 + +/** Build a clean Response from a thrown value, logging the real cause server-side. */ +function storageError(label: string, err: unknown): Response { + console.error(`openvisio import: ${label}`, err) + const detail = err instanceof Error ? err.message : String(err) + // 502, not 500: the function ran fine; its upstream (Blobs) is what failed. + return Response.json({ error: `${label}: ${detail}` }, { status: 502 }) +} + +/** + * Core request handler, parameterized over the blob store so it can be exercised + * without a live Netlify Blobs backend. The default export wires in the real store. + */ +export async function handleImport(req: Request, store: Store): Promise { + const origin = new URL(req.url).origin + + if (req.method === 'POST') { + const body = await req.text() + if (Buffer.byteLength(body) > MAX_BYTES) { + return Response.json({ error: 'graph too large for upload (> 6 MB)' }, { status: 413 }) + } + try { + JSON.parse(body) // reject junk early + } catch { + return Response.json({ error: 'body is not valid JSON' }, { status: 400 }) + } + const id = randomUUID().replace(/-/g, '').slice(0, 10) + try { + await store.set(id, body, { metadata: { name: req.headers.get('x-openvisio-name') ?? '', ts: Date.now() } }) + } catch (err) { + return storageError('storage write failed', err) + } + return Response.json({ id, url: `${origin}/?g=${id}` }, { status: 201 }) + } + + if (req.method === 'GET') { + const id = new URL(req.url).searchParams.get('id') + if (!id) return Response.json({ error: 'missing id' }, { status: 400 }) + let data: string | null + try { + data = await store.get(id, { type: 'text' }) + } catch (err) { + return storageError('storage read failed', err) + } + if (!data) return Response.json({ error: 'graph not found or expired' }, { status: 404 }) + return new Response(data, { + headers: { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=300' }, + }) + } + + return new Response('method not allowed', { status: 405 }) +} + +export default async (req: Request): Promise => { + let store: Store + try { + // getStore is lazy today, but guard it anyway: if the deploy has no Blobs + // context wired up, this is the first place it can surface — and an opaque + // 500 here is exactly the failure we're trying to make legible. + store = getStore('graphs') + } catch (err) { + return storageError('storage unavailable', err) + } + return handleImport(req, store) +} + +// Netlify Functions v2: route this function directly at /api/import. +export const config = { path: '/api/import' } diff --git a/viewer/netlify/import.test.mts b/viewer/netlify/import.test.mts new file mode 100644 index 0000000..98f64c6 --- /dev/null +++ b/viewer/netlify/import.test.mts @@ -0,0 +1,119 @@ +// Branch-coverage test for the transport import handler. Runs with `tsx` and an +// injected fake store — no live Netlify Blobs needed. +// node_modules/.bin/tsx viewer/netlify/import.test.mts +import assert from 'node:assert/strict' +import { handleImport } from './functions/import.mts' + +type Meta = { name: string; ts: number } + +/** In-memory stand-in for a Netlify Blobs Store (only the methods we use). */ +function memStore() { + const data = new Map() + return { + data, + async set(id: string, body: string, _opts?: { metadata?: Meta }) { + data.set(id, body) + }, + async get(id: string, _opts?: { type: 'text' }) { + return data.has(id) ? data.get(id)! : null + }, + } as any +} + +/** A store whose ops always throw — simulates Blobs being unavailable. */ +function brokenStore() { + return { + async set() { + throw new Error('MissingBlobsEnvironmentError: blobs not configured') + }, + async get() { + throw new Error('MissingBlobsEnvironmentError: blobs not configured') + }, + } as any +} + +const post = (body: string, headers: Record = {}) => + new Request('https://openvisio.io/api/import', { method: 'POST', body, headers }) +const get = (qs: string) => new Request(`https://openvisio.io/api/import${qs}`, { method: 'GET' }) + +let passed = 0 +async function test(name: string, fn: () => Promise) { + await fn() + passed++ + console.log(` ok ${name}`) +} + +const graph = JSON.stringify({ name: 'demo', files: [], symbols: [], edges: [] }) + +await test('POST valid graph → 201 with id + url, persisted', async () => { + const store = memStore() + const res = await handleImport(post(graph, { 'x-openvisio-name': 'demo' }), store) + assert.equal(res.status, 201) + const json = (await res.json()) as { id: string; url: string } + assert.match(json.id, /^[0-9a-f]{10}$/) + assert.equal(json.url, `https://openvisio.io/?g=${json.id}`) + assert.equal(store.data.get(json.id), graph) +}) + +await test('POST oversized body → 413 (byte-accurate)', async () => { + // 6 MB + 1 byte of ASCII. + const huge = 'x'.repeat(6 * 1024 * 1024 + 1) + const res = await handleImport(post(huge), memStore()) + assert.equal(res.status, 413) +}) + +await test('POST multibyte body just over the byte cap → 413 (length would miss it)', async () => { + // 'é' is 1 UTF-16 code unit but 2 bytes. (3 Mi + 1) chars = ~6 MB + 2 bytes: + // a naive string.length check (< 6Mi) would WRONGLY accept this. + const MAX_LEN = 6 * 1024 * 1024 + const s = 'é'.repeat(3 * 1024 * 1024 + 1) + assert.ok(s.length < MAX_LEN, 'precondition: length is under the old char limit') + assert.ok(Buffer.byteLength(s) > 6 * 1024 * 1024, 'precondition: bytes exceed the cap') + const res = await handleImport(post(s), memStore()) + assert.equal(res.status, 413) +}) + +await test('POST invalid JSON → 400', async () => { + const res = await handleImport(post('not json {'), memStore()) + assert.equal(res.status, 400) +}) + +await test('POST when blob write throws → 502 with detail, not 500', async () => { + const res = await handleImport(post(graph), brokenStore()) + assert.equal(res.status, 502) + const json = (await res.json()) as { error: string } + assert.match(json.error, /storage write failed/) + assert.match(json.error, /MissingBlobsEnvironmentError/) +}) + +await test('GET round-trips a stored graph → 200 with original body', async () => { + const store = memStore() + const created = (await (await handleImport(post(graph), store)).json()) as { id: string } + const res = await handleImport(get(`?id=${created.id}`), store) + assert.equal(res.status, 200) + assert.equal(res.headers.get('content-type'), 'application/json') + assert.equal(await res.text(), graph) +}) + +await test('GET without id → 400', async () => { + const res = await handleImport(get(''), memStore()) + assert.equal(res.status, 400) +}) + +await test('GET unknown id → 404', async () => { + const res = await handleImport(get('?id=deadbeef00'), memStore()) + assert.equal(res.status, 404) +}) + +await test('GET when blob read throws → 502, not 500', async () => { + const res = await handleImport(get('?id=whatever00'), brokenStore()) + assert.equal(res.status, 502) +}) + +await test('PUT → 405', async () => { + const req = new Request('https://openvisio.io/api/import', { method: 'PUT' }) + const res = await handleImport(req, memStore()) + assert.equal(res.status, 405) +}) + +console.log(`\n${passed} passed`) diff --git a/viewer/package.json b/viewer/package.json index e34b341..32b54ad 100644 --- a/viewer/package.json +++ b/viewer/package.json @@ -1,6 +1,6 @@ { "name": "openvisio-viewer", - "version": "0.2.0", + "version": "0.2.1", "description": "Self-contained Atlas + City graph viewer for OpenVisio — the same React/Three.js views from the app, served locally by `openvisio view`.", "license": "MIT", "type": "module", @@ -31,6 +31,10 @@ "three-stdlib": "^2.35.0", "typescript": "^5.7.2", "vite": "^6.0.0", - "zod": "^3.23.8" + "zod": "^3.23.8", + "web-tree-sitter": "~0.25.10", + "@openvisio/core": "*", + "path-browserify": "^1.0.1", + "@netlify/blobs": "^8.1.0" } } diff --git a/viewer/public/wasm/tree-sitter-bash.wasm b/viewer/public/wasm/tree-sitter-bash.wasm new file mode 100755 index 0000000..214d0a7 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-bash.wasm differ diff --git a/viewer/public/wasm/tree-sitter-c.wasm b/viewer/public/wasm/tree-sitter-c.wasm new file mode 100755 index 0000000..ceda238 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-c.wasm differ diff --git a/viewer/public/wasm/tree-sitter-c_sharp.wasm b/viewer/public/wasm/tree-sitter-c_sharp.wasm new file mode 100755 index 0000000..5c11b4f Binary files /dev/null and b/viewer/public/wasm/tree-sitter-c_sharp.wasm differ diff --git a/viewer/public/wasm/tree-sitter-cpp.wasm b/viewer/public/wasm/tree-sitter-cpp.wasm new file mode 100755 index 0000000..2d453db Binary files /dev/null and b/viewer/public/wasm/tree-sitter-cpp.wasm differ diff --git a/viewer/public/wasm/tree-sitter-css.wasm b/viewer/public/wasm/tree-sitter-css.wasm new file mode 100755 index 0000000..24f8a26 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-css.wasm differ diff --git a/viewer/public/wasm/tree-sitter-dart.wasm b/viewer/public/wasm/tree-sitter-dart.wasm new file mode 100755 index 0000000..17007b4 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-dart.wasm differ diff --git a/viewer/public/wasm/tree-sitter-elisp.wasm b/viewer/public/wasm/tree-sitter-elisp.wasm new file mode 100755 index 0000000..98a7243 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-elisp.wasm differ diff --git a/viewer/public/wasm/tree-sitter-elixir.wasm b/viewer/public/wasm/tree-sitter-elixir.wasm new file mode 100755 index 0000000..e4537eb Binary files /dev/null and b/viewer/public/wasm/tree-sitter-elixir.wasm differ diff --git a/viewer/public/wasm/tree-sitter-elm.wasm b/viewer/public/wasm/tree-sitter-elm.wasm new file mode 100755 index 0000000..97c6a30 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-elm.wasm differ diff --git a/viewer/public/wasm/tree-sitter-embedded_template.wasm b/viewer/public/wasm/tree-sitter-embedded_template.wasm new file mode 100755 index 0000000..8b61793 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-embedded_template.wasm differ diff --git a/viewer/public/wasm/tree-sitter-go.wasm b/viewer/public/wasm/tree-sitter-go.wasm new file mode 100755 index 0000000..a20aba8 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-go.wasm differ diff --git a/viewer/public/wasm/tree-sitter-html.wasm b/viewer/public/wasm/tree-sitter-html.wasm new file mode 100755 index 0000000..50a0092 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-html.wasm differ diff --git a/viewer/public/wasm/tree-sitter-java.wasm b/viewer/public/wasm/tree-sitter-java.wasm new file mode 100755 index 0000000..45022a9 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-java.wasm differ diff --git a/viewer/public/wasm/tree-sitter-javascript.wasm b/viewer/public/wasm/tree-sitter-javascript.wasm new file mode 100755 index 0000000..edaeba9 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-javascript.wasm differ diff --git a/viewer/public/wasm/tree-sitter-json.wasm b/viewer/public/wasm/tree-sitter-json.wasm new file mode 100755 index 0000000..7abe88c Binary files /dev/null and b/viewer/public/wasm/tree-sitter-json.wasm differ diff --git a/viewer/public/wasm/tree-sitter-kotlin.wasm b/viewer/public/wasm/tree-sitter-kotlin.wasm new file mode 100755 index 0000000..b0e4db6 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-kotlin.wasm differ diff --git a/viewer/public/wasm/tree-sitter-lua.wasm b/viewer/public/wasm/tree-sitter-lua.wasm new file mode 100755 index 0000000..6783ea0 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-lua.wasm differ diff --git a/viewer/public/wasm/tree-sitter-objc.wasm b/viewer/public/wasm/tree-sitter-objc.wasm new file mode 100755 index 0000000..8a347a6 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-objc.wasm differ diff --git a/viewer/public/wasm/tree-sitter-ocaml.wasm b/viewer/public/wasm/tree-sitter-ocaml.wasm new file mode 100755 index 0000000..6105e8e Binary files /dev/null and b/viewer/public/wasm/tree-sitter-ocaml.wasm differ diff --git a/viewer/public/wasm/tree-sitter-php.wasm b/viewer/public/wasm/tree-sitter-php.wasm new file mode 100755 index 0000000..505fe86 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-php.wasm differ diff --git a/viewer/public/wasm/tree-sitter-python.wasm b/viewer/public/wasm/tree-sitter-python.wasm new file mode 100755 index 0000000..1423763 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-python.wasm differ diff --git a/viewer/public/wasm/tree-sitter-ql.wasm b/viewer/public/wasm/tree-sitter-ql.wasm new file mode 100755 index 0000000..ffe8224 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-ql.wasm differ diff --git a/viewer/public/wasm/tree-sitter-r.wasm b/viewer/public/wasm/tree-sitter-r.wasm new file mode 100644 index 0000000..4ce49a9 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-r.wasm differ diff --git a/viewer/public/wasm/tree-sitter-rescript.wasm b/viewer/public/wasm/tree-sitter-rescript.wasm new file mode 100755 index 0000000..5c17ef7 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-rescript.wasm differ diff --git a/viewer/public/wasm/tree-sitter-ruby.wasm b/viewer/public/wasm/tree-sitter-ruby.wasm new file mode 100755 index 0000000..2713e11 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-ruby.wasm differ diff --git a/viewer/public/wasm/tree-sitter-rust.wasm b/viewer/public/wasm/tree-sitter-rust.wasm new file mode 100755 index 0000000..5b3b213 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-rust.wasm differ diff --git a/viewer/public/wasm/tree-sitter-scala.wasm b/viewer/public/wasm/tree-sitter-scala.wasm new file mode 100755 index 0000000..e23c24f Binary files /dev/null and b/viewer/public/wasm/tree-sitter-scala.wasm differ diff --git a/viewer/public/wasm/tree-sitter-solidity.wasm b/viewer/public/wasm/tree-sitter-solidity.wasm new file mode 100755 index 0000000..42c1ceb Binary files /dev/null and b/viewer/public/wasm/tree-sitter-solidity.wasm differ diff --git a/viewer/public/wasm/tree-sitter-swift.wasm b/viewer/public/wasm/tree-sitter-swift.wasm new file mode 100755 index 0000000..87282f2 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-swift.wasm differ diff --git a/viewer/public/wasm/tree-sitter-systemrdl.wasm b/viewer/public/wasm/tree-sitter-systemrdl.wasm new file mode 100755 index 0000000..a90cf49 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-systemrdl.wasm differ diff --git a/viewer/public/wasm/tree-sitter-tlaplus.wasm b/viewer/public/wasm/tree-sitter-tlaplus.wasm new file mode 100755 index 0000000..914aac4 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-tlaplus.wasm differ diff --git a/viewer/public/wasm/tree-sitter-toml.wasm b/viewer/public/wasm/tree-sitter-toml.wasm new file mode 100755 index 0000000..f5d6e44 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-toml.wasm differ diff --git a/viewer/public/wasm/tree-sitter-tsx.wasm b/viewer/public/wasm/tree-sitter-tsx.wasm new file mode 100755 index 0000000..1e11feb Binary files /dev/null and b/viewer/public/wasm/tree-sitter-tsx.wasm differ diff --git a/viewer/public/wasm/tree-sitter-typescript.wasm b/viewer/public/wasm/tree-sitter-typescript.wasm new file mode 100755 index 0000000..36c7ae0 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-typescript.wasm differ diff --git a/viewer/public/wasm/tree-sitter-vue.wasm b/viewer/public/wasm/tree-sitter-vue.wasm new file mode 100755 index 0000000..32fa568 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-vue.wasm differ diff --git a/viewer/public/wasm/tree-sitter-yaml.wasm b/viewer/public/wasm/tree-sitter-yaml.wasm new file mode 100755 index 0000000..d9a609a Binary files /dev/null and b/viewer/public/wasm/tree-sitter-yaml.wasm differ diff --git a/viewer/public/wasm/tree-sitter-zig.wasm b/viewer/public/wasm/tree-sitter-zig.wasm new file mode 100755 index 0000000..5a8f381 Binary files /dev/null and b/viewer/public/wasm/tree-sitter-zig.wasm differ diff --git a/viewer/public/wasm/tree-sitter.wasm b/viewer/public/wasm/tree-sitter.wasm new file mode 100755 index 0000000..10916b8 Binary files /dev/null and b/viewer/public/wasm/tree-sitter.wasm differ diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx index ace7fa3..4b4f1e6 100644 --- a/viewer/src/App.tsx +++ b/viewer/src/App.tsx @@ -10,6 +10,7 @@ import { CityView } from '@/components/city/CityView' import { IndexingDialog } from '@/components/workspace/IndexingDialog' import { IndexingProgress } from '@/components/workspace/IndexingProgress' import { GraphResponseSchema, type GraphResponse } from '@/lib/api/types' +import { indexGithubRepo, parseGithubUrl, type FetchProgress } from '@/indexer/github' import { cn } from '@/lib/utils' type Mode = 'city' | 'atlas' @@ -27,6 +28,13 @@ const MODES: ModeDef[] = [ { id: 'city', label: 'City', icon: Building2, hint: 'Your code as a city' }, ] +// Human label for the in-browser GitHub fetch/index phases. +function progressLabel(p: FetchProgress): string { + if (p.phase === 'tree') return 'Reading repository tree…' + if (p.phase === 'fetch') return `Fetching files… ${p.loaded ?? 0}/${p.total ?? '?'}` + return 'Parsing in your browser…' +} + export function App() { const [source, setSource] = useState(() => new URLSearchParams(location.search).get('path') ?? '') const [graph, setGraph] = useState(null) @@ -35,21 +43,31 @@ export function App() { const [loading, setLoading] = useState(false) const [error, setError] = useState(null) const [dialogOpen, setDialogOpen] = useState(false) + const [progress, setProgress] = useState(null) const [agent, setAgent] = useState<'off' | 'idle' | 'live'>('off') const [agentTool, setAgentTool] = useState('') // No AI narrator here, so nothing is cited — the views take an empty map. const citations = useMemo(() => new Map(), []) - const indexRepo = useCallback(async (repoPath: string) => { - if (!repoPath) return + const indexRepo = useCallback(async (input: string) => { + if (!input) return setLoading(true) setError(null) + setProgress(null) try { - const res = await fetch('/api/graph?path=' + encodeURIComponent(repoPath)) - const body = await res.json() - if (!res.ok) throw new Error(body?.error ?? 'index failed (' + res.status + ')') - setGraph(GraphResponseSchema.parse(body.graph)) + let g: GraphResponse + if (parseGithubUrl(input)) { + // GitHub repo → fetch + index entirely in the browser (no server). + g = await indexGithubRepo(input, { onProgress: (p) => setProgress(progressLabel(p)) }) + } else { + // Local path → the `openvisio view` server indexes it (CLI mode). + const res = await fetch('/api/graph?path=' + encodeURIComponent(input)) + const body = await res.json() + if (!res.ok) throw new Error(body?.error ?? 'index failed (' + res.status + ')') + g = GraphResponseSchema.parse(body.graph) + } + setGraph(g) setFocusedFileId(null) setDialogOpen(false) // success — drop the dialog and reveal the map } catch (err) { @@ -57,14 +75,34 @@ export function App() { setGraph(null) } finally { setLoading(false) + setProgress(null) } }, []) + // A transported graph (?g=): fetch the pre-computed graph the CLI uploaded + // and render it — no indexing here, the work already happened on the machine. + const transportedId = useMemo(() => new URLSearchParams(location.search).get('g'), []) + useEffect(() => { + if (!transportedId) return + setLoading(true) + setError(null) + fetch('/api/import?id=' + encodeURIComponent(transportedId)) + .then(async (r) => { + const body = await r.json() + if (!r.ok) throw new Error(body?.error ?? 'graph not found (' + r.status + ')') + setGraph(GraphResponseSchema.parse(body.graph)) + setDialogOpen(false) + }) + .catch((err) => setError(err instanceof Error ? err.message : String(err))) + .finally(() => setLoading(false)) + }, [transportedId]) + // Index whatever ?path= we booted with (or the user re-indexes via the dialog). useEffect(() => { if (source) void indexRepo(source) }, [source, indexRepo]) - // Open the index dialog by hand when there's no repo yet (first run, no ?path=). - useEffect(() => { if (!source) setDialogOpen(true) }, [source]) + // Open the index dialog by hand when there's nothing to show yet (no ?path=, + // no transported ?g=). + useEffect(() => { if (!source && !transportedId) setDialogOpen(true) }, [source, transportedId]) const startIndex = useCallback((repoPath: string) => { const p = repoPath.trim() @@ -175,7 +213,7 @@ export function App() {
{loading && !dialogOpen ? (
- setLoading(false)} /> + setLoading(false)} />
) : error ? (
Error: {error}
diff --git a/viewer/src/components/workspace/IndexingDialog.tsx b/viewer/src/components/workspace/IndexingDialog.tsx index e5bc776..72c4d7f 100644 --- a/viewer/src/components/workspace/IndexingDialog.tsx +++ b/viewer/src/components/workspace/IndexingDialog.tsx @@ -103,7 +103,7 @@ export function IndexingDialog({ open, initialPath, busy, error, onStart, onCanc ) : ( <>

- The path is read in place — nothing is copied or sent anywhere. + A GitHub URL is fetched + indexed in your browser. A local path is read in + place by the CLI — nothing is copied or sent anywhere.

)} diff --git a/viewer/src/indexer/github.ts b/viewer/src/indexer/github.ts new file mode 100644 index 0000000..c58b79b --- /dev/null +++ b/viewer/src/indexer/github.ts @@ -0,0 +1,118 @@ +// Browser GitHub fetcher. Turns a repo URL into an in-memory file set with NO +// clone and NO server: one CORS-friendly call to the GitHub trees API for the +// file list, then each file pulled from raw.githubusercontent (a CDN — CORS-ok +// and not subject to the API's 60/hr anon rate limit). Feeds indexFiles(). +// +// Caps (reasoned from fetch time + browser memory): the bottleneck is the N raw +// requests, so we bound files/bytes to keep an index under ~2 min; bigger repos +// are told to use `openvisio transport` (the local CLI). + +import { indexFiles, type InputFile } from './index' +import type { GraphResponse } from '@/lib/api/types' + +export const CAP_FILES = 1500 +export const CAP_TOTAL_BYTES = 20 * 1024 * 1024 +export const CAP_FILE_BYTES = 1.5 * 1024 * 1024 + +const BINARY_EXT = new Set([ + 'png','jpg','jpeg','gif','bmp','ico','webp','tiff','svg','avif','heic','pdf','mp4','mov','avi','webm','mkv', + 'mp3','wav','flac','ogg','m4a','woff','woff2','ttf','otf','eot','zip','tar','gz','tgz','bz2','xz','zst','rar','7z', + 'jar','war','class','so','dylib','dll','exe','bin','dat','wasm','node','pyc','parquet','db','sqlite','lock', +]) +const EXCLUDE_DIR = /(^|\/)(node_modules|\.git|dist|build|out|\.next|\.turbo|coverage|vendor|target|__pycache__|\.venv|venv)(\/|$)/ + +export interface RepoRef { owner: string; repo: string; ref?: string } +export interface FetchProgress { phase: 'tree' | 'fetch' | 'index'; loaded?: number; total?: number } +export interface FetchOptions { + token?: string + onProgress?: (p: FetchProgress) => void + signal?: AbortSignal +} + +/** Parse the common GitHub URL/`owner/repo` forms → {owner, repo, ref?}. */ +export function parseGithubUrl(input: string): RepoRef | null { + let s = input.trim().replace(/\.git$/, '') + const m = s.match(/github\.com[/:]([^/]+)\/([^/]+)(?:\/tree\/([^/]+(?:\/[^?#]*)?))?/i) + if (m) return { owner: m[1], repo: m[2], ref: m[3] } + const short = s.match(/^([\w.-]+)\/([\w.-]+)$/) // owner/repo + if (short) return { owner: short[1], repo: short[2] } + return null +} + +const extOf = (p: string) => p.slice(p.lastIndexOf('.') + 1).toLowerCase() + +async function ghApi(path: string, opts: FetchOptions): Promise { + const headers: Record = { Accept: 'application/vnd.github+json' } + if (opts.token) headers.Authorization = `Bearer ${opts.token}` + return fetch(`https://api.github.com${path}`, { headers, signal: opts.signal }) +} + +interface TreeEntry { path: string; type: string; size?: number } + +/** Resolve files + contents for a repo. Throws a readable error on failure. */ +export async function fetchRepoFiles(input: string, opts: FetchOptions = {}): Promise<{ files: InputFile[]; repoName: string; truncated: boolean; skipped: number }> { + const ref0 = parseGithubUrl(input) + if (!ref0) throw new Error('Not a GitHub repo URL (expected github.com/owner/repo or owner/repo)') + opts.onProgress?.({ phase: 'tree' }) + + // Default branch if none given. + let resolvedRef = ref0.ref + if (!resolvedRef) { + const r = await ghApi(`/repos/${ref0.owner}/${ref0.repo}`, opts) + if (r.status === 404) throw new Error('Repo not found (private repos need a token)') + if (!r.ok) throw new Error(`GitHub API ${r.status}${r.status === 403 ? ' — rate limited; add a token' : ''}`) + resolvedRef = ((await r.json()) as { default_branch?: string }).default_branch + } + const ref: string = resolvedRef || 'main' + + // One recursive tree call → the whole file list. + const tr = await ghApi(`/repos/${ref0.owner}/${ref0.repo}/git/trees/${encodeURIComponent(ref)}?recursive=1`, opts) + if (!tr.ok) throw new Error(`Could not read repo tree (${tr.status}${tr.status === 403 ? ' — rate limited; add a token' : ''})`) + const tree = await tr.json() + const truncated = Boolean(tree.truncated) + + // Filter to text/source blobs under the caps. + let total = 0, skipped = 0 + const picks: TreeEntry[] = [] + for (const e of (tree.tree ?? []) as TreeEntry[]) { + if (e.type !== 'blob') continue + if (EXCLUDE_DIR.test(e.path) || BINARY_EXT.has(extOf(e.path))) { skipped++; continue } + if ((e.size ?? 0) > CAP_FILE_BYTES) { skipped++; continue } + if (picks.length >= CAP_FILES || total + (e.size ?? 0) > CAP_TOTAL_BYTES) { + throw new Error(`Repo too large for in-browser indexing (> ${CAP_FILES} files / ${CAP_TOTAL_BYTES / 1e6}MB). Use \`openvisio transport\` for this one.`) + } + picks.push(e); total += e.size ?? 0 + } + + // Pull each file from the raw CDN, concurrency-limited. + const files: InputFile[] = new Array(picks.length) + let done = 0 + opts.onProgress?.({ phase: 'fetch', loaded: 0, total: picks.length }) + const base = `https://raw.githubusercontent.com/${ref0.owner}/${ref0.repo}/${ref}/` + const CONCURRENCY = 12 + let next = 0 + async function worker() { + while (next < picks.length) { + const i = next++ + const e = picks[i] + try { + const res = await fetch(base + e.path.split('/').map(encodeURIComponent).join('/'), { signal: opts.signal }) + files[i] = { path: e.path, content: res.ok ? await res.text() : '' } + } catch { + files[i] = { path: e.path, content: '' } + } + done++ + if (done % 8 === 0 || done === picks.length) opts.onProgress?.({ phase: 'fetch', loaded: done, total: picks.length }) + } + } + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, picks.length) }, worker)) + + return { files, repoName: ref0.repo, truncated, skipped } +} + +/** Fetch + index a GitHub repo entirely in the browser → GraphResponse. */ +export async function indexGithubRepo(input: string, opts: FetchOptions = {}): Promise { + const { files, repoName } = await fetchRepoFiles(input, opts) + opts.onProgress?.({ phase: 'index' }) + return indexFiles(files, repoName) +} diff --git a/viewer/src/indexer/index.ts b/viewer/src/indexer/index.ts new file mode 100644 index 0000000..a7003ec --- /dev/null +++ b/viewer/src/indexer/index.ts @@ -0,0 +1,171 @@ +// In-browser indexer. Takes an in-memory file list ({path, content}) and +// produces the viewer's GraphResponse — no server, no clone. Reuses @openvisio/ +// core's actual grammar configs (all ~37 languages: queries + keep/exported/ +// importSpecifier/resolveImport) via the browser-safe `@openvisio/core/browser` +// entry; only the I/O is browser-flavoured (fetch wasm + in-memory files). The +// per-file parse loop mirrors core's parseFile so output matches the CLI engine. + +import { GRAMMARS, grammarIdFromPath, type GrammarId } from '@openvisio/core/browser' +import { loadGrammar, parse, getQueries } from './treesitter' +import type { Node } from 'web-tree-sitter' +import type { GraphResponse, Language } from '@/lib/api/types' + +export interface InputFile { + path: string // repo-relative POSIX + content: string +} + +// Heavy grammar disabled in the engine too (large wasm). Its files become plain +// nodes without parsed symbols. +const DISABLED = new Set(['swift']) + +// grammar id → display language (most are identity; these two differ). +const GRAMMAR_LANG: Partial> = { tsx: 'typescript', c_sharp: 'csharp' } +// Light language labels for common non-parsed files, so they still read sensibly. +const EXT_LANG: Record = { + json: 'json', yaml: 'yaml', yml: 'yaml', toml: 'toml', md: 'markdown', mdx: 'markdown', + html: 'html', htm: 'html', css: 'css', vue: 'vue', +} +const extOf = (p: string) => p.slice(p.lastIndexOf('.') + 1).toLowerCase() +const countLoc = (s: string) => s.split('\n').filter((l) => l.trim().length > 0).length + +function languageOf(path: string, grammar: GrammarId | null): Language { + if (grammar) return GRAMMAR_LANG[grammar] ?? (grammar as Language) + return EXT_LANG[extOf(path)] ?? 'other' +} + +// Mirror of core/parse/index.ts signatureOf — single-line elided declaration. +function signatureOf(def: Node): string { + const text = def.text + let end = text.length + const brace = text.indexOf('{') + if (brace !== -1) end = Math.min(end, brace) + const nl = text.indexOf('\n') + if (nl !== -1) end = Math.min(end, nl) + const sig = text.slice(0, end).replace(/\s+/g, ' ').trim() + return sig.length > 200 ? sig.slice(0, 197) + '…' : sig +} + +interface ParsedSymbol { name: string; kind: string; signature: string; startLine: number; endLine: number; exported: boolean } +interface ParsedFile { + id: number + path: string + language: Language + grammar: GrammarId | null + loc: number + symbols: ParsedSymbol[] + imports: string[] +} + +/** Parse one file → symbols + import specifiers. Mirrors core's parseFile. */ +async function parseOne(id: number, file: InputFile): Promise { + const grammar = grammarIdFromPath(file.path) + const pf: ParsedFile = { id, path: file.path, language: languageOf(file.path, grammar), grammar, loc: countLoc(file.content), symbols: [], imports: [] } + if (!grammar || DISABLED.has(grammar)) return pf + const config = GRAMMARS[grammar] + if (!config) return pf + await loadGrammar(grammar) + let root: Node + try { root = parse(grammar, file.content) } catch { return pf } + + const q = getQueries(grammar, config) + const seen = new Set() + for (const match of q.symbolQuery.matches(root)) { + let nameNode: Node | undefined, defNode: Node | undefined, captureName = '' + for (const cap of match.captures) { + if (cap.name === 'name') nameNode = cap.node + else if (cap.name.startsWith('def.')) { defNode = cap.node; captureName = cap.name } + } + if (!nameNode || !defNode) continue + const name = nameNode.text + if (!config.keep(defNode, name)) continue + const key = `${name}:${defNode.startPosition.row}` + if (seen.has(key)) continue + seen.add(key) + pf.symbols.push({ + name, + kind: captureName.slice('def.'.length), + signature: signatureOf(defNode), + startLine: defNode.startPosition.row + 1, + endLine: defNode.endPosition.row + 1, + exported: config.exported(defNode, name), + }) + } + pf.symbols.sort((a, b) => a.startLine - b.startLine) + + if (q.importQuery) { + const specSeen = new Set() + for (const match of q.importQuery.matches(root)) { + for (const cap of match.captures) { + const spec = config.importSpecifier(cap.node).trim() + if (spec.length > 0 && !specSeen.has(spec)) { specSeen.add(spec); pf.imports.push(spec) } + } + } + } + return pf +} + +/** Index an in-memory file set → GraphResponse for the viewer. */ +export async function indexFiles(files: InputFile[], repoName = 'repo'): Promise { + // Pre-load the grammars present so parse calls are ready (and fetched once). + const needed = new Set() + for (const f of files) { const g = grammarIdFromPath(f.path); if (g && !DISABLED.has(g)) needed.add(g) } + await Promise.all([...needed].map((g) => loadGrammar(g).catch((e) => console.warn('[indexer] grammar', g, 'failed', e)))) + + const parsed: ParsedFile[] = [] + let id = 0 + for (const f of files) { parsed.push(await parseOne(id, f)); id++ } + + // Build the viewer GraphResponse. Import edges use each grammar's own + // resolveImport against the set of repo-relative paths. + const byPath = new Map() + const bySet = new Set() + for (const pf of parsed) { byPath.set(pf.path, pf.id); bySet.add(pf.path) } + + const symbols: GraphResponse['symbols'] = [] + let sid = 0 + for (const pf of parsed) for (const s of pf.symbols) { + symbols.push({ id: sid++, file_id: pf.id, name: s.name, kind: s.kind as GraphResponse['symbols'][number]['kind'], start_line: s.startLine, end_line: s.endLine, is_exported: s.exported }) + } + + const edges: GraphResponse['edges'] = [] + let eid = 0 + for (const pf of parsed) { + if (!pf.grammar) continue + const config = GRAMMARS[pf.grammar] + if (!config) continue + const targets = new Map() // targetId → weight (dedup multi-imports) + for (const spec of pf.imports) { + let resolved: string | null = null + try { resolved = config.resolveImport(pf.path, spec, bySet) } catch { resolved = null } + if (!resolved) continue + const target = byPath.get(resolved) + if (target == null || target === pf.id) continue + targets.set(target, (targets.get(target) ?? 0) + 1) + } + for (const [target, weight] of targets) { + edges.push({ id: eid++, source_id: pf.id, target_id: target, source_kind: 'file', target_kind: 'file', edge_kind: 'import', weight }) + } + } + + const langCounts = new Map() + let totalLoc = 0 + for (const pf of parsed) { langCounts.set(pf.language, (langCounts.get(pf.language) ?? 0) + 1); totalLoc += pf.loc } + + return { + repo: { + repo_id: 'browser-' + repoName, + name: repoName, + root_path: repoName, + file_count: parsed.length, + total_loc: totalLoc, + languages: [...langCounts.entries()].sort((a, b) => b[1] - a[1]).map(([language, file_count]) => ({ language, file_count })), + indexed_at: 0, + }, + files: parsed.map((pf) => ({ id: pf.id, repo_id: 'browser-' + repoName, path: pf.path, language: pf.language, loc: pf.loc, sha: '', last_modified: 0 })), + symbols, + edges, + history: [], + coupling: [], + } +} diff --git a/viewer/src/indexer/treesitter.ts b/viewer/src/indexer/treesitter.ts new file mode 100644 index 0000000..076f31c --- /dev/null +++ b/viewer/src/indexer/treesitter.ts @@ -0,0 +1,74 @@ +// Browser tree-sitter loader (PoC). The SAME web-tree-sitter the Node engine +// uses — only the I/O differs: grammar wasm comes from a fetched URL instead of +// the filesystem (`Language.load` already takes bytes), and the runtime wasm is +// located by URL via `Parser.init({ locateFile })`. Grammars lazy-load per +// detected language, so a repo only pulls the few MB it needs. + +import { Parser, Language, Query, type Node } from 'web-tree-sitter' + +const WASM_BASE = '/wasm' +let initPromise: Promise | null = null +const grammars = new Map() + +/** Initialise the tree-sitter runtime once (idempotent). */ +function ensureInit(): Promise { + if (!initPromise) { + initPromise = Parser.init({ + locateFile: (file: string) => (file.endsWith('.wasm') ? `${WASM_BASE}/tree-sitter.wasm` : file), + // Matches the Node engine: 30+ grammars + the query compiler overrun the + // default 32 MB heap. + INITIAL_MEMORY: 256 * 1024 * 1024, + }) + } + return initPromise +} + +/** Fetch + load a grammar wasm by id (e.g. 'typescript'). Cached. */ +export async function loadGrammar(id: string): Promise { + await ensureInit() + if (grammars.has(id)) return + const res = await fetch(`${WASM_BASE}/tree-sitter-${id}.wasm`) + if (!res.ok) throw new Error(`grammar ${id} wasm fetch failed (${res.status})`) + const bytes = new Uint8Array(await res.arrayBuffer()) + grammars.set(id, await Language.load(bytes)) +} + +/** Parse source with a loaded grammar → syntax-tree root node. */ +export function parse(id: string, source: string): Node { + const lang = grammars.get(id) + if (!lang) throw new Error(`grammar ${id} not loaded`) + const parser = new Parser() + parser.setLanguage(lang) + const tree = parser.parse(source) + if (!tree) throw new Error(`parse failed for grammar ${id}`) + return tree.rootNode +} + +/** Compile a tree-sitter query against a loaded grammar. */ +export function compileQuery(id: string, source: string): Query { + const lang = grammars.get(id) + if (!lang) throw new Error(`grammar ${id} not loaded`) + return new Query(lang, source) +} + +export interface CompiledQueries { + symbolQuery: Query + importQuery: Query | null + callQuery: Query | null +} +const queryCache = new Map() + +/** Compile + cache a grammar's queries (mirror of core's getOrCompileQueries). */ +export function getQueries(id: string, config: { symbolQuery: string; importQuery: string | null; callQuery?: string }): CompiledQueries { + const key = `${id}::${config.symbolQuery}::${config.importQuery}::${config.callQuery ?? ''}` + let cached = queryCache.get(key) + if (!cached) { + cached = { + symbolQuery: compileQuery(id, config.symbolQuery), + importQuery: config.importQuery ? compileQuery(id, config.importQuery) : null, + callQuery: config.callQuery ? compileQuery(id, config.callQuery) : null, + } + queryCache.set(key, cached) + } + return cached +} diff --git a/viewer/src/main.tsx b/viewer/src/main.tsx index ec84a09..ab8ea15 100644 --- a/viewer/src/main.tsx +++ b/viewer/src/main.tsx @@ -1,7 +1,14 @@ import { createRoot } from 'react-dom/client' import './styles.css' import { App } from './App' +import { indexFiles } from './indexer' +import { indexGithubRepo } from './indexer/github' // No StrictMode: it double-invokes effects in dev, which would spin up two // EventSource/WebGL contexts. The build is the product here, not a dev session. createRoot(document.getElementById('root')!).render() + +// Dev/test hooks: prove the engine indexes in the browser. `openvisioIndex(files)` +// runs web-tree-sitter on in-memory files; `openvisioIndexGithub(url)` fetches a +// repo from GitHub (no server) and indexes it. Both return the GraphResponse. +Object.assign(window as object, { openvisioIndex: indexFiles, openvisioIndexGithub: indexGithubRepo }) diff --git a/viewer/src/shims/node-path.ts b/viewer/src/shims/node-path.ts new file mode 100644 index 0000000..1072966 --- /dev/null +++ b/viewer/src/shims/node-path.ts @@ -0,0 +1,7 @@ +// node:path shim for the browser. The engine's grammar configs use `path.posix` +// for import resolution; in the browser everything is posix, so we back it with +// path-browserify and expose `.posix` (which path-browserify itself omits). +import p from 'path-browserify' +export const posix = p +export const { normalize, join, dirname, basename, extname, relative, resolve, isAbsolute, sep } = p +export default p diff --git a/viewer/src/shims/path-browserify.d.ts b/viewer/src/shims/path-browserify.d.ts new file mode 100644 index 0000000..e570b4e --- /dev/null +++ b/viewer/src/shims/path-browserify.d.ts @@ -0,0 +1,15 @@ +declare module 'path-browserify' { + interface PosixPath { + normalize(p: string): string + join(...parts: string[]): string + dirname(p: string): string + basename(p: string, ext?: string): string + extname(p: string): string + relative(from: string, to: string): string + resolve(...parts: string[]): string + isAbsolute(p: string): boolean + sep: string + } + const path: PosixPath + export default path +} diff --git a/viewer/vite.config.ts b/viewer/vite.config.ts index dcec856..979b273 100644 --- a/viewer/vite.config.ts +++ b/viewer/vite.config.ts @@ -9,7 +9,12 @@ import * as path from 'node:path' export default defineConfig({ plugins: [react(), tailwindcss()], resolve: { - alias: { '@': path.resolve(__dirname, 'src') }, + alias: { + '@': path.resolve(__dirname, 'src'), + // The engine's grammar configs import `node:path` (path.posix) for import + // resolution — shim it so they run in the browser bundle. + 'node:path': path.resolve(__dirname, 'src/shims/node-path.ts'), + }, }, base: './', // assets referenced relative, so the static server can host at / build: {