Skip to content

Lazy-load the file tree, decouple the workspace index #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
evanklem wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Lazy-load the file tree, decouple the workspace index #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/host/src/plugin-loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,8 @@ function permissionForMethod(method: string): HostPermission | null {
case 'knowledge.write':
return 'knowledge.write';
case 'editor.tree':
case 'editor.listDir':
case 'editor.listFiles':
case 'editor.open':
case 'editor.read':
return 'editor.read';
Expand Down
40 changes: 40 additions & 0 deletions packages/host/src/rpc-server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,24 @@ export type FileTreeNode =
| { kind: 'file'; name: string; path: string; subtitle?: string }
| { kind: 'folder'; name: string; children: FileTreeNode[] };

/* no-adapter fallbacks for the lazy editor.listDir / editor.listFiles handlers,
so headless hosts and tests still respond off the in-memory fileTree. */
function sliceTreeLevel(tree: FileTreeNode[], path: string): FileTreeNode[] {
const segments = path.split('/').filter(Boolean);
let level = tree;
for (const segment of segments) {
const folder = level.find((node) => node.kind === 'folder' && node.name === segment);
if (!folder || folder.kind !== 'folder') return [];
level = folder.children;
}
// collapse folders at this level — children are resolved on demand
return level.map((node) => (node.kind === 'folder' ? { kind: 'folder', name: node.name, children: [] } : node));
}

function flattenTreeFiles(tree: FileTreeNode[]): string[] {
return tree.flatMap((node) => (node.kind === 'file' ? [node.path] : flattenTreeFiles(node.children)));
}

export type SkillRecord = {
id: string;
name: string;
Expand Down Expand Up @@ -297,6 +315,8 @@ export type ExternalOpener = (url: string) => Promise<boolean> | boolean;
export type EditorSearchMatch = { file: string; line: number; text: string };
export type FileSystemAdapter = {
listTree?: () => Promise<FileTreeNode[]>;
listDir?: (path: string) => Promise<FileTreeNode[]>;
listFiles?: () => Promise<string[]>;
readText?: (path: string) => Promise<string>;
writeText?: (path: string, content: string) => Promise<void>;
search?: (opts: { query: string; regex?: boolean; glob?: string; limit?: number }) => Promise<EditorSearchMatch[]>;
Expand Down Expand Up @@ -1565,6 +1585,24 @@ export class HostRpcServer {
}
return { tree: this.fileTree };
});
/* lazy one-level listing — the explorer resolves a folder's children on
expand instead of walking the whole tree up front. without an adapter we
slice the in-memory fileTree so tests/headless hosts still respond. */
this.registerHandler('editor.listDir', async (params) => {
const { path } = params as { path: string };
if (this.fileSystemAdapter?.listDir) {
return { tree: await this.fileSystemAdapter.listDir(path) };
}
return { tree: sliceTreeLevel(this.fileTree, path) };
});
/* complete workspace file index — feeds quick-open and cross-file
resolution, kept separate from the lazily-loaded explorer tree. */
this.registerHandler('editor.listFiles', async () => {
if (this.fileSystemAdapter?.listFiles) {
return { files: await this.fileSystemAdapter.listFiles() };
}
return { files: flattenTreeFiles(this.fileTree) };
});
this.registerHandler('editor.open', (params) => {
const { path } = params as { path: string };
this.publish('editor:opened', { path });
Expand Down Expand Up @@ -2640,6 +2678,8 @@ const RPC_PARAM_VALIDATORS: Record<string, ParamValidator> = {
'ui.openExternal': (params) => validateRef('https://polypore.dev/schemas/rpc/ui.schema.json#/definitions/ui.openExternal.params', params),
'state.get': objectShape({ key: { required: true, check: (value) => stringValue(value) && STATE_KEYS.has(value), message: 'key must be a known state key' } }),
'editor.tree': emptyParams,
'editor.listDir': objectShape({ path: { required: true, check: stringValue, message: 'path must be a string' } }),
'editor.listFiles': emptyParams,
'editor.open': objectShape({
path: { required: true, check: stringValue, message: 'path must be a string' },
opts: { check: looseObject, message: 'opts must be an object' },
Expand Down
4 changes: 4 additions & 0 deletions packages/sdk/src/host.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ export interface PolyporeHost {
};
editor: {
tree(): Promise<{ tree: FileTreeNode[] }>;
listDir(path: string): Promise<{ tree: FileTreeNode[] }>;
listFiles(): Promise<{ files: string[] }>;
open(path: string, opts?: { line?: number; col?: number }): Promise<{ opened: boolean; path: string }>;
onOpen(fn: (event: { path: string }) => void): Unsubscribe;
read(path: string): Promise<{ path: string; content: string }>;
Expand Down Expand Up @@ -683,6 +685,8 @@ export function createLoopbackHost(
},
editor: {
tree: () => call<{ tree: FileTreeNode[] }>('editor.tree', {}),
listDir: (path) => call<{ tree: FileTreeNode[] }>('editor.listDir', { path }),
listFiles: () => call<{ files: string[] }>('editor.listFiles', {}),
open: (path, opts) => call<{ opened: boolean; path: string }>('editor.open', { path, opts }),
onOpen: (fn) => subscribe?.('editor:opened', (payload) => fn(payload as { path: string })) ?? (() => {}),
read: (path) => call<{ path: string; content: string }>('editor.read', { path }),
Expand Down
131 changes: 104 additions & 27 deletions plugins/editor/component.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
const [contextMenu, setContextMenu] = useState<{ x: number; y: number; info: FileTreeContextInfo } | null>(null);
const [query, setQuery] = useState('');
const [tree, setTree] = useState<FileNode[]>([]);
const [indexFiles, setIndexFiles] = useState<string[]>([]);
const [fileText, setFileText] = useState('');
const [monacoReady, setMonacoReady] = useState(false);
const [diagnostics, setDiagnostics] = useState<Diagnostic[]>([]);
Expand Down Expand Up @@ -258,23 +259,15 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
the yield, this fires synchronously with all the other editor
useEffects after commit and the panel can't paint until the
chain drains. with it, the editor frame paints instantly and the
file tree fills in on the next task. */
file tree fills in on the next task. only the root level is fetched;
a folder's children are resolved lazily on expand (loadChildren). */
let cancelled = false;
const cancelSchedule = scheduleAfterPaint(() => {
if (cancelled) return;
perfPoint('editor:tree-effect-fires');
host.editor.tree().then((result) => {
host.editor.listDir('').then((result) => {
if (cancelled || !result.tree) return;
const nextTree = result.tree as FileNode[];
setTree(nextTree);
const nextFiles = flattenFiles(nextTree);
setActiveFile((current) => {
if (current && nextFiles.includes(current)) return current;
if (nextFiles.length === 0) return '';
const preferred = nextFiles.find((path) => path.toLowerCase().endsWith('readme.md')) ?? nextFiles[0];
setOpenFiles([preferred]);
return preferred;
});
setTree(result.tree as FileNode[]);
}).catch(() => {
setTree([]);
});
Expand All @@ -285,6 +278,70 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
};
}, [host]);

/* workspace file index — complete and gitignore-aware, kept separate from the
lazily-loaded explorer tree so quick-open and Monaco cross-file resolution
always see every file. also drives the initial active-file pick. */
useEffect(() => {
let cancelled = false;
host.editor.listFiles().then((result) => {
if (cancelled || !result.files) return;
setIndexFiles(result.files);
setActiveFile((current) => {
if (current && result.files.includes(current)) return current;
if (result.files.length === 0) return '';
const preferred = pickDefaultFile(result.files);
setOpenFiles([preferred]);
return preferred;
});
}).catch(() => {
if (!cancelled) setIndexFiles([]);
});
return () => { cancelled = true; };
}, [host]);

/* mirror tree into a ref so async refreshes can read which folders were
expanded without taking `tree` as a dependency (would re-bind callbacks
on every keystroke-driven re-render). */
const treeRef = useRef<FileNode[]>([]);
useEffect(() => { treeRef.current = tree; }, [tree]);

/* resolve one folder's children on first expand and merge them in place. */
const loadChildren = useCallback((folderPath: string) => {
host.editor.listDir(folderPath).then((result) => {
if (!result.tree) return;
setTree((current) => mergeChildren(current, folderPath, result.tree as FileNode[]));
}).catch(() => {});
}, [host]);

const refreshIndex = useCallback(() => {
host.editor.listFiles()
.then((result) => setIndexFiles(result.files ?? []))
.catch(() => {});
}, [host]);

/* re-fetch exactly the levels that were previously loaded so a create/delete/
rename refreshes the tree without collapsing the user's expanded folders. */
const reloadTree = useCallback(async () => {
const rebuild = async (folderPath: string, prev: FileNode[]): Promise<FileNode[]> => {
const result = await host.editor.listDir(folderPath);
const fresh = (result.tree ?? []) as FileNode[];
return Promise.all(fresh.map(async (node) => {
if (node.kind !== 'folder') return node;
const prevFolder = prev.find(
(p): p is Extract<FileNode, { kind: 'folder' }> => p.kind === 'folder' && p.name === node.name,
);
if (!prevFolder?.loaded) return node;
const childPath = folderPath ? `${folderPath}/${node.name}` : node.name;
return { ...node, children: await rebuild(childPath, prevFolder.children), loaded: true };
}));
};
try {
setTree(await rebuild('', treeRef.current));
} catch {
/* leave the existing tree in place if a refresh fails */
}
}, [host]);

useEffect(() => {
let cancelled = false;
host.editor.read(LANGUAGE_SERVERS_CONFIG_PATH).then((result) => {
Expand Down Expand Up @@ -336,7 +393,7 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
window.clearTimeout(timeout);
};
}, [activeFile, fileText, host, monacoReady]);
const allFiles = useMemo(() => flattenFiles(tree), [tree]);
const allFiles = indexFiles;
const isDirty = activeFile !== '' && fileText !== savedText;
const newEntryDir = newFilePath.includes('/') ? newFilePath.slice(0, newFilePath.lastIndexOf('/') + 1) : '';
const diagnosticsByFile = useMemo(() => {
Expand Down Expand Up @@ -937,8 +994,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
setNewFileOpen(false);
setNewFilePath('');
openFile(path, { preserveStatus: true });
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
setSavedText('');
setFileText('');
setSaveStatus('created');
Expand Down Expand Up @@ -970,8 +1027,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
await host.fs.mkdir(path);
setNewFileOpen(false);
setNewFilePath('');
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
setSaveStatus('folder created');
} catch (err) {
setFileActionError(err instanceof Error ? err.message : 'could not create folder');
Expand All @@ -990,8 +1047,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
await host.fs.mkdir(path);
setPlusPopupOpen(false);
setPlusName('');
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
setSaveStatus('folder created');
} catch (err) {
setPlusError(err instanceof Error ? err.message : 'could not create folder');
Expand All @@ -1003,8 +1060,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
setPlusPopupOpen(false);
setPlusName('');
openFile(path, { preserveStatus: true });
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
setSavedText('');
setFileText('');
setSaveStatus('created');
Expand Down Expand Up @@ -1034,8 +1091,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
setOpenFiles(next);
if (activeFile === path || activeFile.startsWith(prefix)) setActiveFile(next[0] ?? '');
}
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
} catch (err) {
setSaveStatus(err instanceof Error ? err.message : 'delete failed');
}
Expand All @@ -1058,8 +1115,8 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
}));
if (activeFile === path) setActiveFile(newPath);
else if (activeFile.startsWith(prefix)) setActiveFile(`${newPath}/${activeFile.slice(prefix.length)}`);
const result = await host.editor.tree();
setTree(result.tree as FileNode[]);
await reloadTree();
refreshIndex();
} catch (err) {
setSaveStatus(err instanceof Error ? err.message : 'rename failed');
}
Expand Down Expand Up @@ -1170,6 +1227,7 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
nodes={tree}
activePath={activeFile}
onSelect={openFile}
onExpand={loadChildren}
metaFor={(path) => {
const m = fileMeta(path);
return m ? { status: m.status, diagnostics: m.diagnostics } : undefined;
Expand Down Expand Up @@ -1487,8 +1545,27 @@ export function EditorPanel({ header, host }: BuiltinPluginProps) {
);
}

function flattenFiles(nodes: FileNode[]): string[] {
return nodes.flatMap((node) => node.kind === 'file' ? [node.path] : flattenFiles(node.children));
/* binary/asset extensions to skip when auto-opening a default file — the index
(rg --files) is complete and includes them, but they'd render as garbage text. */
const BINARY_ASSET_EXT = /\.(png|jpe?g|gif|webp|ico|bmp|svg|pdf|zip|gz|tar|woff2?|ttf|otf|eot|mp[34]|mov|webm|wasm|so|dylib|dll|exe|bin)$/i;

/* prefer a readme, else the first editable (non-binary) file, else anything. */
function pickDefaultFile(files: string[]): string {
const readme = files.find((path) => path.toLowerCase().endsWith('readme.md'));
if (readme) return readme;
return files.find((path) => !BINARY_ASSET_EXT.test(path)) ?? files[0] ?? '';
}

/* merge a lazily-resolved level into the tree at folderPath, marking the folder
loaded so it isn't re-fetched on a later expand. */
function mergeChildren(nodes: FileNode[], folderPath: string, children: FileNode[]): FileNode[] {
const segments = folderPath.split('/').filter(Boolean);
const [head, ...rest] = segments;
return nodes.map((node) => {
if (node.kind !== 'folder' || node.name !== head) return node;
if (rest.length === 0) return { ...node, children, loaded: true };
return { ...node, children: mergeChildren(node.children, rest.join('/'), children) };
});
}

function normalizeProjectLanguageConfig(value: unknown): ProjectLanguageConfig {
Expand Down
23 changes: 21 additions & 2 deletions plugins/shared/file-tree.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ export type FileTreeProps = {
metaFor?: (path: string) => FileMeta | undefined;
basePath?: string;
onContextMenu?: (e: React.MouseEvent, info: FileTreeContextInfo) => void;
/* opt-in lazy loading: when set, a folder's children are requested via
onExpand(folderPath) on first expand instead of being walked up front.
Omit it for eager trees whose children are already fully resolved. */
onExpand?: (folderPath: string) => void;
};

export function FileTree({
Expand All @@ -34,6 +38,7 @@ export function FileTree({
metaFor,
basePath = '',
onContextMenu,
onExpand,
}: FileTreeProps) {
return (
<div className="file-tree" role={depth === 0 ? 'tree' : 'group'}>
Expand All @@ -51,6 +56,7 @@ export function FileTree({
metaFor={metaFor}
basePath={basePath}
onContextMenu={onContextMenu}
onExpand={onExpand}
/>
) : (
<FileTreeFile
Expand Down Expand Up @@ -82,6 +88,7 @@ function FileTreeFolder({
metaFor,
basePath,
onContextMenu,
onExpand,
}: {
node: Extract<FileNode, { kind: 'folder' }>;
depth: number;
Expand All @@ -93,9 +100,20 @@ function FileTreeFolder({
metaFor?: (path: string) => FileMeta | undefined;
basePath: string;
onContextMenu?: (e: React.MouseEvent, info: FileTreeContextInfo) => void;
onExpand?: (folderPath: string) => void;
}) {
const [open, setOpen] = useState(defaultExpanded);
const folderPath = basePath ? `${basePath}/${node.name}` : node.name;
// lazy mode reveals child count only once the level has been resolved
const showCount = !onExpand || node.loaded;

const toggle = () => {
setOpen((v) => {
const next = !v;
if (next && onExpand && !node.loaded) onExpand(folderPath);
return next;
});
};

return (
<div className="file-tree__folder" role="treeitem" aria-expanded={open}>
Expand All @@ -104,7 +122,7 @@ function FileTreeFolder({
className="file-tree__row file-tree__row--folder"
style={{ paddingLeft: 6 + depth * 12 }}
title={folderPath}
onClick={() => setOpen((v) => !v)}
onClick={toggle}
onContextMenu={
onContextMenu
? (e) => {
Expand All @@ -118,7 +136,7 @@ function FileTreeFolder({
<span className="file-tree__chevron" aria-hidden="true">{open ? 'v' : '>'}</span>
<span className="file-tree__icon file-tree__icon--folder" aria-hidden="true" />
<span className="file-tree__label">{node.name}</span>
<small className="file-tree__count">{node.children.length}</small>
{showCount && <small className="file-tree__count">{node.children.length}</small>}
</button>
{open && (
<FileTree
Expand All @@ -132,6 +150,7 @@ function FileTreeFolder({
metaFor={metaFor}
basePath={folderPath}
onContextMenu={onContextMenu}
onExpand={onExpand}
/>
)}
</div>
Expand Down
Loading
Loading