From acdbbb23fc527dcced3fd36186f94c7df7686132 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:34:15 +0200 Subject: [PATCH 01/11] feat: add xtap pool setup wizard --- .github/workflows/ci.yml | 10 ++- README.md | 16 ++-- package-lock.json | 57 ++++++++++--- package.json | 8 +- setup/package.json | 24 ++++++ setup/src/config.ts | 58 +++++++++++++ setup/src/deploy.ts | 89 ++++++++++++++++++++ setup/src/hub-api.ts | 72 ++++++++++++++++ setup/src/main.ts | 19 +++++ setup/src/process.ts | 56 +++++++++++++ setup/src/root.ts | 18 ++++ setup/src/stage.ts | 56 +++++++++++++ setup/src/token.ts | 142 ++++++++++++++++++++++++++++++++ setup/src/wizard.ts | 150 ++++++++++++++++++++++++++++++++++ setup/tests/config.test.ts | 41 ++++++++++ setup/tests/hub-api.test.ts | 62 ++++++++++++++ setup/tests/process.test.ts | 24 ++++++ setup/tests/root.test.ts | 28 +++++++ setup/tests/stage.test.ts | 73 +++++++++++++++++ setup/tests/token.test.ts | 77 +++++++++++++++++ setup/tsconfig.json | 10 +++ setup/tsconfig.typecheck.json | 10 +++ setup/vitest.config.ts | 19 +++++ slophammer.yml | 8 ++ vitest.config.ts | 18 +++- 25 files changed, 1120 insertions(+), 25 deletions(-) create mode 100644 setup/package.json create mode 100644 setup/src/config.ts create mode 100644 setup/src/deploy.ts create mode 100644 setup/src/hub-api.ts create mode 100644 setup/src/main.ts create mode 100644 setup/src/process.ts create mode 100644 setup/src/root.ts create mode 100644 setup/src/stage.ts create mode 100644 setup/src/token.ts create mode 100644 setup/src/wizard.ts create mode 100644 setup/tests/config.test.ts create mode 100644 setup/tests/hub-api.test.ts create mode 100644 setup/tests/process.test.ts create mode 100644 setup/tests/root.test.ts create mode 100644 setup/tests/stage.test.ts create mode 100644 setup/tests/token.test.ts create mode 100644 setup/tsconfig.json create mode 100644 setup/tsconfig.typecheck.json create mode 100644 setup/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a6930d..5b7b44b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,15 @@ jobs: npm run coverage --workspace explorer npm run dry --workspace explorer npm run mutate --workspace explorer - - run: npm run build --workspace explorer && npm run build --workspace space + - name: Check setup + run: | + npm run format --workspace setup + npm run lint --workspace setup + npm run typecheck --workspace setup + npm run test --workspace setup + npm run coverage --workspace setup + npm run dry --workspace setup + - run: npm run build --workspace explorer && npm run build --workspace space && npm run build --workspace setup - name: Slophammer uses: dutifuldev/slophammer@main with: diff --git a/README.md b/README.md index 1d81b07..addaf1b 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,23 @@ design and delivery plan. ## Set up a pool (once, by the pool owner) -Requires a personal `hf auth login` with write access to the target namespace: +Requires Node.js 22+, npm, the Hugging Face CLI, and a personal `hf auth login` +with write access to the target namespace: ```sh -scripts/deploy-space.sh # creates the dataset repo + Docker Space, sets secrets +npm ci +npm run setup ``` -The script prints the two remaining manual steps: storing a fine-grained -`HF_TOKEN` Space secret (write access to the one dataset repo) and optionally -seeding an existing xtap-store archive: +The setup flow creates or updates the private dataset repo and public Docker +Space, configures the Space variables and generated secrets, verifies the +dataset-only `HF_TOKEN`, and can seed an existing xtap-store archive. + +The lower-level scripts are still available when you want to do those steps +manually: ```sh +scripts/deploy-space.sh scripts/seed-dataset.sh /xtap-pool-data ~/xtap-store/data/tweets ``` diff --git a/package-lock.json b/package-lock.json index 8623060..b4aca8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "workspaces": [ "shared", "space", - "explorer" + "explorer", + "setup" ], "devDependencies": { "@eslint/js": "^9.39.2", @@ -723,6 +724,27 @@ "specificity": "bin/cli.js" } }, + "node_modules/@clack/core": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@clack/core/-/core-0.5.0.tgz", + "integrity": "sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow==", + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "sisteransi": "^1.0.5" + } + }, + "node_modules/@clack/prompts": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@clack/prompts/-/prompts-0.11.0.tgz", + "integrity": "sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==", + "license": "MIT", + "dependencies": { + "@clack/core": "0.5.0", + "picocolors": "^1.0.0", + "sisteransi": "^1.0.5" + } + }, "node_modules/@csstools/color-helpers": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", @@ -3466,6 +3488,10 @@ "resolved": "explorer", "link": true }, + "node_modules/@xtap-pool/setup": { + "resolved": "setup", + "link": true + }, "node_modules/@xtap-pool/shared": { "resolved": "shared", "link": true @@ -5602,7 +5628,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5624,7 +5649,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5646,7 +5670,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5668,7 +5691,6 @@ "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5690,7 +5712,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5712,7 +5733,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5734,7 +5754,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5756,7 +5775,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5778,7 +5796,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5800,7 +5817,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -5822,7 +5838,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -6325,7 +6340,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -6900,6 +6914,12 @@ "simple-concat": "^1.0.0" } }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, "node_modules/slophammer-ts": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/slophammer-ts/-/slophammer-ts-0.4.0.tgz", @@ -7970,6 +7990,17 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "setup": { + "name": "@xtap-pool/setup", + "version": "0.1.0", + "dependencies": { + "@clack/prompts": "^0.11.0", + "@huggingface/hub": "^2.13.2" + }, + "bin": { + "xtap-pool-setup": "dist/src/main.js" + } + }, "shared": { "name": "@xtap-pool/shared", "version": "0.1.0", diff --git a/package.json b/package.json index f06db76..ebce5d3 100644 --- a/package.json +++ b/package.json @@ -6,19 +6,21 @@ "workspaces": [ "shared", "space", - "explorer" + "explorer", + "setup" ], "scripts": { + "setup": "npm run dev --workspace setup", "format": "prettier --check .", "format:write": "prettier --write .", "lint": "eslint .", - "typecheck": "tsc -p tsconfig.json --noEmit && tsc --build shared space explorer", + "typecheck": "tsc -p tsconfig.json --noEmit && tsc --build shared space explorer setup", "test": "vitest run", "test:extension": "node --test extension/tests/*.test.mjs", "coverage": "vitest run --coverage", "dry": "slophammer-ts dry .", "mutate": "stryker run", - "build": "npm run build --workspace explorer && npm run build --workspace space", + "build": "npm run build --workspace explorer && npm run build --workspace space && npm run build --workspace setup", "check": "npm run format && npm run lint && npm run typecheck && npm test && npm run test:extension && npm run test:extension:native && npm run coverage && npm run dry", "test:extension:native": "uvx pytest extension/tests -q" }, diff --git a/setup/package.json b/setup/package.json new file mode 100644 index 0000000..e408b5e --- /dev/null +++ b/setup/package.json @@ -0,0 +1,24 @@ +{ + "name": "@xtap-pool/setup", + "version": "0.1.0", + "private": true, + "type": "module", + "bin": { + "xtap-pool-setup": "dist/src/main.js" + }, + "scripts": { + "build": "tsc --build", + "dev": "node --experimental-strip-types src/main.ts", + "start": "node dist/src/main.js", + "format": "prettier --check .", + "lint": "eslint .", + "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", + "test": "vitest run", + "coverage": "vitest run --coverage", + "dry": "cd .. && slophammer-ts dry ." + }, + "dependencies": { + "@clack/prompts": "^0.11.0", + "@huggingface/hub": "^2.13.2" + } +} diff --git a/setup/src/config.ts b/setup/src/config.ts new file mode 100644 index 0000000..48ab7da --- /dev/null +++ b/setup/src/config.ts @@ -0,0 +1,58 @@ +export type SetupConfig = { + namespace: string; + spaceRepo: string; + datasetRepo: string; + allowedUsers: readonly string[]; +}; + +const REPO_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._-]*$/; +const USERNAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +export function defaultSetupConfig(username: string): SetupConfig { + return { + namespace: username, + spaceRepo: `${username}/xtap-pool`, + datasetRepo: `${username}/xtap-pool-data`, + allowedUsers: [username], + }; +} + +export function normalizeUsers(input: string): readonly string[] { + return [ + ...new Set( + input + .split(",") + .map((user) => user.trim()) + .filter((user) => user.length > 0), + ), + ]; +} + +export function usersValue(users: readonly string[]): string { + return users.join(","); +} + +export function repoInNamespace(namespace: string, repoName: string): string { + return `${namespace}/${repoName}`; +} + +export function validateRepoId(value: string): string | undefined { + return REPO_ID.test(value) ? undefined : "Use owner/name, for example osolmaz/xtap-pool."; +} + +export function validateUserList(value: string): string | undefined { + const users = normalizeUsers(value); + if (users.length === 0) return "Enter at least one Hugging Face username."; + return users.every((user) => USERNAME.test(user)) + ? undefined + : "Use comma-separated Hugging Face usernames."; +} + +export function spacePublicUrl(spaceRepo: string): string { + const [namespace = "", name = ""] = spaceRepo.split("/"); + return `https://${namespace}-${name}.hf.space`; +} + +export function tokenSettingsUrl(): string { + return "https://huggingface.co/settings/tokens/new?tokenType=fineGrained"; +} diff --git a/setup/src/deploy.ts b/setup/src/deploy.ts new file mode 100644 index 0000000..2ecc10c --- /dev/null +++ b/setup/src/deploy.ts @@ -0,0 +1,89 @@ +import { randomBytes } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createRepo, repoExists, uploadFiles } from "@huggingface/hub"; + +import type { SetupConfig } from "./config.js"; +import { usersValue } from "./config.js"; +import type { HubClient } from "./hub-api.js"; +import { getSpaceVariables, setSpaceSecret, setSpaceVariable } from "./hub-api.js"; +import { captureCommand } from "./process.js"; +import { collectUploadFiles, createSpaceStage } from "./stage.js"; + +export async function deployPool( + root: string, + client: HubClient, + config: SetupConfig, +): Promise { + await ensureRepo(client, { type: "dataset", name: config.datasetRepo }, "private"); + await ensureRepo(client, { type: "space", name: config.spaceRepo }, "public"); + await uploadSpace(root, client, config.spaceRepo); + await configureSpace(client, config); +} + +export async function configureSpace(client: HubClient, config: SetupConfig): Promise { + const variables = await getSpaceVariables(client, config.spaceRepo); + await setSpaceVariable(client, config.spaceRepo, "DATASET_REPO", config.datasetRepo); + await setSpaceVariable( + client, + config.spaceRepo, + "ALLOWED_USERS", + usersValue(config.allowedUsers), + ); + if (!variables.has("SECRETS_INITIALIZED")) { + await setSpaceSecret(client, config.spaceRepo, "POOL_SIGNING_SECRET", randomSecret()); + await setSpaceSecret(client, config.spaceRepo, "SESSION_SECRET", randomSecret()); + await setSpaceVariable(client, config.spaceRepo, "SECRETS_INITIALIZED", "1"); + } +} + +async function ensureRepo( + client: HubClient, + repo: { type: "dataset" | "space"; name: string }, + visibility: "private" | "public", +): Promise { + const exists = await repoExists({ + repo, + accessToken: client.accessToken, + ...fetchOption(client), + }); + if (exists) return; + await createRepo({ + repo, + accessToken: client.accessToken, + visibility, + ...(repo.type === "space" ? { sdk: "docker" as const } : {}), + ...fetchOption(client), + }); +} + +async function uploadSpace(root: string, client: HubClient, spaceRepo: string): Promise { + const stageDir = await mkdtemp(join(tmpdir(), "xtap-pool-space-")); + try { + await createSpaceStage(root, stageDir); + await uploadFiles({ + repo: { type: "space", name: spaceRepo }, + accessToken: client.accessToken, + files: [...(await collectUploadFiles(stageDir))], + commitTitle: `deploy: ${await shortHead(root)}`, + ...fetchOption(client), + }); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +} + +async function shortHead(root: string): Promise { + const result = await captureCommand("git", ["-C", root, "rev-parse", "--short", "HEAD"]); + return result.stdout.trim(); +} + +function randomSecret(): string { + return randomBytes(32).toString("hex"); +} + +function fetchOption(client: HubClient): { fetch?: typeof fetch } { + return client.fetchFn === undefined ? {} : { fetch: client.fetchFn }; +} diff --git a/setup/src/hub-api.ts b/setup/src/hub-api.ts new file mode 100644 index 0000000..28bba3d --- /dev/null +++ b/setup/src/hub-api.ts @@ -0,0 +1,72 @@ +import { HUB_URL } from "@huggingface/hub"; + +export type HubClient = { + accessToken: string; + hubUrl?: string; + fetchFn?: typeof fetch; +}; + +type JsonObject = Record; + +export async function getSpaceVariables( + client: HubClient, + spaceRepo: string, +): Promise> { + const payload = await hubJson(client, `/api/spaces/${spaceRepo}/variables`, { method: "GET" }); + return parseSpaceVariables(payload); +} + +export async function setSpaceVariable( + client: HubClient, + spaceRepo: string, + key: string, + value: string, +): Promise { + await hubJson(client, `/api/spaces/${spaceRepo}/variables`, { + method: "POST", + body: JSON.stringify({ key, value }), + }); +} + +export async function setSpaceSecret( + client: HubClient, + spaceRepo: string, + key: string, + value: string, +): Promise { + await hubJson(client, `/api/spaces/${spaceRepo}/secrets`, { + method: "POST", + body: JSON.stringify({ key, value }), + }); +} + +export function parseSpaceVariables(payload: unknown): ReadonlyMap { + const result = new Map(); + for (const [key, value] of Object.entries(asRecord(payload))) { + const variable = asRecord(value); + if (typeof value === "string") result.set(key, value); + else if (typeof variable["value"] === "string") result.set(key, variable["value"]); + } + return result; +} + +async function hubJson(client: HubClient, path: string, init: RequestInit): Promise { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${client.accessToken}`); + headers.set("content-type", "application/json"); + const response = await (client.fetchFn ?? fetch)(`${client.hubUrl ?? HUB_URL}${path}`, { + ...init, + headers, + }); + if (!response.ok) { + throw new Error(`Hub request failed (${String(response.status)}): ${await response.text()}`); + } + const payload: unknown = await response.json(); + return asRecord(payload); +} + +function asRecord(value: unknown): JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonObject) + : {}; +} diff --git a/setup/src/main.ts b/setup/src/main.ts new file mode 100644 index 0000000..0424682 --- /dev/null +++ b/setup/src/main.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { cancel } from "@clack/prompts"; + +import { findProjectRoot } from "./root.js"; +import { runSetupWizard } from "./wizard.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const root = findProjectRoot(here); + +try { + await runSetupWizard(root); +} catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + cancel(message); + process.exit(1); +} diff --git a/setup/src/process.ts b/setup/src/process.ts new file mode 100644 index 0000000..40f81cf --- /dev/null +++ b/setup/src/process.ts @@ -0,0 +1,56 @@ +import { spawn } from "node:child_process"; + +export type CommandResult = { + stdout: string; + stderr: string; +}; + +export type CommandOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; +}; + +export async function captureCommand( + command: string, + args: readonly string[], + options: CommandOptions = {}, +): Promise { + return run(command, args, options, true); +} + +export async function inheritCommand( + command: string, + args: readonly string[], + options: CommandOptions = {}, +): Promise { + await run(command, args, options, false); +} + +async function run( + command: string, + args: readonly string[], + options: CommandOptions, + capture: boolean, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + cwd: options.cwd, + env: options.env, + stdio: capture ? ["ignore", "pipe", "pipe"] : "inherit", + }); + const chunks = { stdout: "", stderr: "" }; + if (capture) { + child.stdout?.on("data", (chunk: Buffer) => { + chunks.stdout += chunk.toString("utf8"); + }); + child.stderr?.on("data", (chunk: Buffer) => { + chunks.stderr += chunk.toString("utf8"); + }); + } + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(chunks); + else reject(new Error(`${command} ${args.join(" ")} exited with ${String(code)}`)); + }); + }); +} diff --git a/setup/src/root.ts b/setup/src/root.ts new file mode 100644 index 0000000..73c4977 --- /dev/null +++ b/setup/src/root.ts @@ -0,0 +1,18 @@ +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; + +export function findProjectRoot(start: string): string { + let current = start; + for (;;) { + if (isProjectRoot(current)) return current; + const parent = dirname(current); + if (parent === current) throw new Error("Could not find xtap-pool project root."); + current = parent; + } +} + +function isProjectRoot(path: string): boolean { + return ( + existsSync(join(path, "package.json")) && existsSync(join(path, "space", "hf-space-README.md")) + ); +} diff --git a/setup/src/stage.ts b/setup/src/stage.ts new file mode 100644 index 0000000..902bac4 --- /dev/null +++ b/setup/src/stage.ts @@ -0,0 +1,56 @@ +import { Blob } from "node:buffer"; +import { promises as fs } from "node:fs"; +import { join, relative, sep } from "node:path"; + +import { captureCommand, inheritCommand } from "./process.js"; + +export type UploadFile = { + path: string; + content: Blob; +}; + +const SPACE_EXCLUDED_ROOTS = new Set(["docs", "extension", "setup"]); + +export async function createSpaceStage(root: string, stageDir: string): Promise { + const archivePath = join(stageDir, "repo.tar"); + await captureCommand("git", ["-C", root, "archive", "--format=tar", "-o", archivePath, "HEAD"]); + await inheritCommand("tar", ["-xf", archivePath, "-C", stageDir]); + await fs.rm(archivePath, { force: true }); + await fs.copyFile(join(root, "space", "hf-space-README.md"), join(stageDir, "README.md")); + await Promise.all( + [...SPACE_EXCLUDED_ROOTS].map((name) => + fs.rm(join(stageDir, name), { recursive: true, force: true }), + ), + ); +} + +export async function collectUploadFiles(root: string): Promise { + const files = await listFiles(root, root); + return Promise.all( + files.filter(shouldUploadPath).map(async (path) => ({ + path, + content: new Blob([await fs.readFile(join(root, path))]), + })), + ); +} + +export function shouldUploadPath(path: string): boolean { + const [first] = path.split("/"); + return first !== undefined && !SPACE_EXCLUDED_ROOTS.has(first) && first !== ".git"; +} + +async function listFiles(root: string, current: string): Promise { + const entries = await fs.readdir(current, { withFileTypes: true }); + const nested = await Promise.all(entries.map((entry) => listEntry(root, current, entry))); + return nested.flat(); +} + +async function listEntry( + root: string, + current: string, + entry: { name: string; isDirectory: () => boolean }, +): Promise { + const absolute = join(current, entry.name); + if (entry.isDirectory()) return listFiles(root, absolute); + return [relative(root, absolute).split(sep).join("/")]; +} diff --git a/setup/src/token.ts b/setup/src/token.ts new file mode 100644 index 0000000..2c4ea59 --- /dev/null +++ b/setup/src/token.ts @@ -0,0 +1,142 @@ +type JsonObject = Record; + +const FINE_GRAINED_ROLE = "fineGrained"; +const TARGET_PERMISSIONS = new Set(["repo.content.read", "repo.content.write"]); +const REQUIRED_WRITE_PERMISSION = "repo.content.write"; + +export type DatasetTokenReport = + | { + ok: true; + username: string; + tokenName: string; + permissions: readonly string[]; + } + | { + ok: false; + errors: readonly string[]; + }; + +export async function verifyDatasetWriteToken(params: { + token: string; + datasetRepo: string; + fetchFn?: typeof fetch; +}): Promise { + const response = await (params.fetchFn ?? fetch)("https://huggingface.co/api/whoami-v2", { + headers: { authorization: `Bearer ${params.token}` }, + }); + if (!response.ok) + return { ok: false, errors: [`Hugging Face rejected the token (${String(response.status)}).`] }; + const payload: unknown = await response.json(); + return evaluateDatasetWriteToken(payload, params.datasetRepo); +} + +export function evaluateDatasetWriteToken( + payload: unknown, + datasetRepo: string, +): DatasetTokenReport { + const root = asRecord(payload); + const accessToken = asRecord(asRecord(root["auth"])["accessToken"]); + const fineGrained = asRecord(accessToken["fineGrained"]); + const role = text(accessToken["role"]); + const errors = + role === FINE_GRAINED_ROLE + ? [] + : [`Token role is '${role || "unknown"}', expected fine-grained.`]; + errors.push(...globalPermissionErrors(fineGrained)); + const targetPermissions = scopedPermissionErrors(fineGrained, datasetRepo, errors); + if (!targetPermissions.has(REQUIRED_WRITE_PERMISSION)) { + errors.push(`Token must include ${REQUIRED_WRITE_PERMISSION} on ${datasetRepo}.`); + } + if (errors.length > 0) return { ok: false, errors }; + return { + ok: true, + username: text(root["name"]), + tokenName: text(accessToken["displayName"]), + permissions: [...targetPermissions].sort(), + }; +} + +function globalPermissionErrors(fineGrained: JsonObject): string[] { + return strings(fineGrained["global"]).map( + (permission) => `Unexpected global permission: ${permission}.`, + ); +} + +function scopedPermissionErrors( + fineGrained: JsonObject, + datasetRepo: string, + errors: string[], +): Set { + const targetPermissions = new Set(); + for (const scope of array(fineGrained["scoped"])) { + collectScopePermissions(asRecord(scope), datasetRepo, targetPermissions, errors); + } + return targetPermissions; +} + +function collectScopePermissions( + scope: JsonObject, + datasetRepo: string, + targetPermissions: Set, + errors: string[], +): void { + const entity = asRecord(scope["entity"]); + for (const permission of strings(scope["permissions"])) { + if (matchesDataset(entity, datasetRepo)) { + recordTargetPermission(permission, targetPermissions, errors); + } else { + errors.push( + `Unexpected permission outside ${datasetRepo}: ${permission} on ${entityLabel(entity)}.`, + ); + } + } +} + +function recordTargetPermission( + permission: string, + targetPermissions: Set, + errors: string[], +): void { + if (TARGET_PERMISSIONS.has(permission)) targetPermissions.add(permission); + else errors.push(`Unexpected permission on dataset token: ${permission}.`); +} + +function matchesDataset(entity: JsonObject, datasetRepo: string): boolean { + return entityCandidates(entity).some((candidate) => normalizeRepo(candidate) === datasetRepo); +} + +function entityCandidates(entity: JsonObject): readonly string[] { + const name = text(entity["name"]); + const namespace = text(entity["namespace"]); + return [text(entity["id"]), name, namespace && name ? `${namespace}/${name}` : ""].filter( + (candidate) => candidate.length > 0, + ); +} + +function entityLabel(entity: JsonObject): string { + const kind = text(entity["type"]) || "unknown"; + const name = text(entity["name"]) || text(entity["id"]) || "unknown"; + return `${kind}:${name}`; +} + +function normalizeRepo(value: string): string { + return value.replace(/^datasets\//, ""); +} + +function asRecord(value: unknown): JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonObject) + : {}; +} + +function array(value: unknown): readonly unknown[] { + return Array.isArray(value) ? value : []; +} + +function strings(value: unknown): readonly string[] { + return array(value).filter((item): item is string => typeof item === "string"); +} + +function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} diff --git a/setup/src/wizard.ts b/setup/src/wizard.ts new file mode 100644 index 0000000..7e61096 --- /dev/null +++ b/setup/src/wizard.ts @@ -0,0 +1,150 @@ +import { + note, + outro, + password, + spinner, + text, + confirm, + intro, + cancel, + isCancel, +} from "@clack/prompts"; +import { whoAmI } from "@huggingface/hub"; + +import type { SetupConfig } from "./config.js"; +import { + defaultSetupConfig, + normalizeUsers, + spacePublicUrl, + tokenSettingsUrl, + usersValue, + validateRepoId, + validateUserList, +} from "./config.js"; +import { deployPool } from "./deploy.js"; +import { setSpaceSecret } from "./hub-api.js"; +import { captureCommand, inheritCommand } from "./process.js"; +import { verifyDatasetWriteToken } from "./token.js"; + +export async function runSetupWizard(root: string): Promise { + intro("xtap-pool setup"); + const accessToken = await activeHfToken(); + const account = await whoAmI({ accessToken }); + const config = await promptConfig(account.name); + await confirmPlan(config); + const task = spinner(); + task.start("Creating repos, deploying Space, and setting generated secrets"); + await deployPool(root, { accessToken }, config); + task.stop("Space deployed"); + const datasetToken = await promptDatasetToken(config.datasetRepo); + await setSpaceSecret({ accessToken }, config.spaceRepo, "HF_TOKEN", datasetToken); + await maybeSeed(root, config); + outro(`Done. Explorer: ${spacePublicUrl(config.spaceRepo)}`); +} + +async function activeHfToken(): Promise { + const result = await captureCommand("hf", ["auth", "token", "--quiet"]); + const token = result.stdout.trim(); + if (token.length === 0) throw new Error("No active hf token. Run `hf auth login` first."); + return token; +} + +async function promptConfig(username: string): Promise { + const defaults = defaultSetupConfig(username); + const namespace = await promptText("Hugging Face namespace", defaults.namespace); + const spaceRepo = await promptText("Space repo", defaults.spaceRepo, validateRepoId); + const datasetRepo = await promptText( + "Private dataset repo", + defaults.datasetRepo, + validateRepoId, + ); + const allowed = await promptText( + "Allowed HF users", + usersValue(defaults.allowedUsers), + validateUserList, + ); + return { + namespace, + spaceRepo, + datasetRepo, + allowedUsers: normalizeUsers(allowed), + }; +} + +async function confirmPlan(config: SetupConfig): Promise { + note( + [ + `Space: ${config.spaceRepo}`, + `Dataset: ${config.datasetRepo}`, + `Allowed users: ${usersValue(config.allowedUsers)}`, + ].join("\n"), + "Plan", + ); + const ok = await confirm({ message: "Create/update these resources?", initialValue: true }); + if (isCancel(ok) || !ok) { + cancel("Setup cancelled."); + process.exit(130); + } +} + +async function promptDatasetToken(datasetRepo: string): Promise { + note( + [ + `Create a fine-grained token with write access only to ${datasetRepo}.`, + tokenSettingsUrl(), + ].join("\n"), + "Dataset token", + ); + for (;;) { + const token = await promptPassword("Paste the dataset-only HF_TOKEN"); + const report = await verifyDatasetWriteToken({ token, datasetRepo }); + if (report.ok) { + note(`${report.tokenName || "token"} on ${report.username || "unknown account"}`, "Verified"); + return token; + } + note(report.errors.join("\n"), "Token refused"); + } +} + +async function maybeSeed(root: string, config: SetupConfig): Promise { + const seed = await confirm({ + message: "Seed an existing xTap archive now?", + initialValue: false, + }); + if (isCancel(seed) || !seed) return; + const username = await promptText( + "Seed data belongs to which HF user?", + config.allowedUsers[0] ?? config.namespace, + ); + const source = await promptText("Existing tweets directory", "~/xtap-store/data/tweets"); + await inheritCommand("scripts/seed-dataset.sh", [config.datasetRepo, username, source], { + cwd: root, + }); +} + +async function promptText( + message: string, + initialValue: string, + validate?: (value: string) => string | undefined, +): Promise { + const value = await text( + validate === undefined ? { message, initialValue } : { message, initialValue, validate }, + ); + if (isCancel(value)) { + cancel("Setup cancelled."); + process.exit(130); + } + return value; +} + +async function promptPassword(message: string): Promise { + const value = await password({ + message, + validate: (input) => (input ? undefined : "Token is required."), + }); + if (isCancel(value)) { + cancel("Setup cancelled."); + process.exit(130); + } + return value; +} diff --git a/setup/tests/config.test.ts b/setup/tests/config.test.ts new file mode 100644 index 0000000..ea7863a --- /dev/null +++ b/setup/tests/config.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; + +import { + defaultSetupConfig, + normalizeUsers, + repoInNamespace, + spacePublicUrl, + tokenSettingsUrl, + usersValue, + validateRepoId, + validateUserList, +} from "../src/config.js"; + +describe("setup config helpers", () => { + it("derives default repos from the active username", () => { + expect(defaultSetupConfig("alice")).toEqual({ + namespace: "alice", + spaceRepo: "alice/xtap-pool", + datasetRepo: "alice/xtap-pool-data", + allowedUsers: ["alice"], + }); + }); + + it("normalizes comma-separated allowlists", () => { + expect(normalizeUsers("alice, bob,alice,, carol ")).toEqual(["alice", "bob", "carol"]); + expect(usersValue(["alice", "bob"])).toBe("alice,bob"); + }); + + it("validates repo ids and user lists", () => { + expect(validateRepoId("alice/xtap-pool")).toBeUndefined(); + expect(validateRepoId("xtap-pool")).toContain("owner/name"); + expect(validateUserList("alice,bob")).toBeUndefined(); + expect(validateUserList("alice, bad user")).toContain("comma-separated"); + }); + + it("formats derived values", () => { + expect(repoInNamespace("alice", "pool")).toBe("alice/pool"); + expect(spacePublicUrl("alice/xtap-pool")).toBe("https://alice-xtap-pool.hf.space"); + expect(tokenSettingsUrl()).toContain("tokenType=fineGrained"); + }); +}); diff --git a/setup/tests/hub-api.test.ts b/setup/tests/hub-api.test.ts new file mode 100644 index 0000000..c954fde --- /dev/null +++ b/setup/tests/hub-api.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { + getSpaceVariables, + parseSpaceVariables, + setSpaceSecret, + setSpaceVariable, +} from "../src/hub-api.js"; + +describe("space variable parsing", () => { + it("accepts current and simple API shapes", () => { + const variables = parseSpaceVariables({ + DATASET_REPO: { value: "alice/xtap-pool-data" }, + ALLOWED_USERS: "alice,bob", + IGNORED: { value: 42 }, + }); + expect([...variables.entries()]).toEqual([ + ["DATASET_REPO", "alice/xtap-pool-data"], + ["ALLOWED_USERS", "alice,bob"], + ]); + }); + + it("sends authenticated variable and secret requests", async () => { + const requests: { url: string; init: RequestInit }[] = []; + const fetchFn: typeof fetch = (input, init) => { + requests.push({ url: requestUrl(input), init: init ?? {} }); + return Promise.resolve(Response.json({ DATASET_REPO: { value: "alice/xtap-pool-data" } })); + }; + const client = { accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn }; + + await expect(getSpaceVariables(client, "alice/xtap-pool")).resolves.toEqual( + new Map([["DATASET_REPO", "alice/xtap-pool-data"]]), + ); + await setSpaceVariable(client, "alice/xtap-pool", "ALLOWED_USERS", "alice,bob"); + await setSpaceSecret(client, "alice/xtap-pool", "HF_TOKEN", "hf_dataset"); + + expect(requests.map((request) => [request.url, request.init.method])).toEqual([ + ["https://hub.test/api/spaces/alice/xtap-pool/variables", "GET"], + ["https://hub.test/api/spaces/alice/xtap-pool/variables", "POST"], + ["https://hub.test/api/spaces/alice/xtap-pool/secrets", "POST"], + ]); + const headers = requests[0]?.init.headers; + expect(headers).toBeInstanceOf(Headers); + expect((headers as Headers).get("authorization")).toBe("Bearer hf_owner"); + }); + + it("surfaces Hub errors with status and body", async () => { + const fetchFn: typeof fetch = () => Promise.resolve(new Response("bad token", { status: 401 })); + await expect( + getSpaceVariables( + { accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn }, + "alice/xtap-pool", + ), + ).rejects.toThrow("Hub request failed (401): bad token"); + }); +}); + +function requestUrl(input: string | URL | Request): string { + if (typeof input === "string") return input; + if (input instanceof URL) return input.toString(); + return input.url; +} diff --git a/setup/tests/process.test.ts b/setup/tests/process.test.ts new file mode 100644 index 0000000..1581bf6 --- /dev/null +++ b/setup/tests/process.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { captureCommand, inheritCommand } from "../src/process.js"; + +describe("process helpers", () => { + it("captures stdout and stderr from a successful command", async () => { + const result = await captureCommand(process.execPath, [ + "-e", + "process.stdout.write('out'); process.stderr.write('err');", + ]); + + expect(result).toEqual({ stdout: "out", stderr: "err" }); + }); + + it("inherits stdio for a successful command", async () => { + await expect(inheritCommand(process.execPath, ["-e", ""])).resolves.toBeUndefined(); + }); + + it("rejects failed commands", async () => { + await expect(captureCommand(process.execPath, ["-e", "process.exit(7);"])).rejects.toThrow( + `${process.execPath} -e process.exit(7); exited with 7`, + ); + }); +}); diff --git a/setup/tests/root.test.ts b/setup/tests/root.test.ts new file mode 100644 index 0000000..f0b6bdc --- /dev/null +++ b/setup/tests/root.test.ts @@ -0,0 +1,28 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { findProjectRoot } from "../src/root.js"; + +describe("project root discovery", () => { + it("walks up from source or dist paths to the repo root", async () => { + const root = join(tmpdir(), `xtap-pool-root-${String(process.pid)}`); + try { + await mkdir(join(root, "setup", "dist", "src"), { recursive: true }); + await mkdir(join(root, "space"), { recursive: true }); + await writeFile(join(root, "package.json"), "{}"); + await writeFile(join(root, "space", "hf-space-README.md"), "space"); + + expect(findProjectRoot(join(root, "setup", "src"))).toBe(root); + expect(findProjectRoot(join(root, "setup", "dist", "src"))).toBe(root); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("throws when no project marker exists", () => { + expect(() => findProjectRoot(tmpdir())).toThrow("Could not find xtap-pool project root."); + }); +}); diff --git a/setup/tests/stage.test.ts b/setup/tests/stage.test.ts new file mode 100644 index 0000000..bc2b7d0 --- /dev/null +++ b/setup/tests/stage.test.ts @@ -0,0 +1,73 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { captureCommand } from "../src/process.js"; +import { collectUploadFiles, createSpaceStage, shouldUploadPath } from "../src/stage.js"; + +describe("space staging helpers", () => { + it("excludes local-only roots from the Space upload", () => { + expect(shouldUploadPath("package.json")).toBe(true); + expect(shouldUploadPath("space/src/server.ts")).toBe(true); + expect(shouldUploadPath("docs/implementation-plan.md")).toBe(false); + expect(shouldUploadPath("extension/manifest.json")).toBe(false); + expect(shouldUploadPath("setup/src/main.ts")).toBe(false); + }); + + it("collects upload files with portable paths", async () => { + const root = join(tmpdir(), `xtap-pool-stage-${String(process.pid)}`); + await mkdir(root, { recursive: true }); + try { + await mkdir(join(root, "space"), { recursive: true }); + await writeFile(join(root, "package.json"), "{}"); + await writeFile(join(root, "space", "app.ts"), "export {};"); + const files = await collectUploadFiles(root); + const names = files.map((file) => file.path).sort(); + expect(names).toEqual(["package.json", "space/app.ts"]); + await expect(readFile(join(root, "package.json"), "utf8")).resolves.toBe("{}"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("stages the Space from the current git commit", async () => { + const repo = join(tmpdir(), `xtap-pool-repo-${String(process.pid)}`); + const stage = join(tmpdir(), `xtap-pool-space-${String(process.pid)}`); + await mkdir(repo, { recursive: true }); + await mkdir(stage, { recursive: true }); + try { + await captureCommand("git", ["init"], { cwd: repo }); + await mkdir(join(repo, "space"), { recursive: true }); + await mkdir(join(repo, "docs"), { recursive: true }); + await mkdir(join(repo, "extension"), { recursive: true }); + await mkdir(join(repo, "setup"), { recursive: true }); + await writeFile(join(repo, "README.md"), "root readme"); + await writeFile(join(repo, "space", "hf-space-README.md"), "space readme"); + await writeFile(join(repo, "space", "server.ts"), "export {};"); + await writeFile(join(repo, "docs", "plan.md"), "internal"); + await writeFile(join(repo, "extension", "manifest.json"), "{}"); + await writeFile(join(repo, "setup", "main.ts"), "export {};"); + await captureCommand("git", ["add", "."], { cwd: repo }); + await captureCommand( + "git", + ["-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "init"], + { + cwd: repo, + }, + ); + + await createSpaceStage(repo, stage); + + await expect(readFile(join(stage, "README.md"), "utf8")).resolves.toBe("space readme"); + await expect(readFile(join(stage, "space", "server.ts"), "utf8")).resolves.toBe("export {};"); + await expect(readdir(join(stage, "docs"))).rejects.toThrow(); + await expect(readdir(join(stage, "extension"))).rejects.toThrow(); + await expect(readdir(join(stage, "setup"))).rejects.toThrow(); + } finally { + await rm(repo, { recursive: true, force: true }); + await rm(stage, { recursive: true, force: true }); + } + }); +}); diff --git a/setup/tests/token.test.ts b/setup/tests/token.test.ts new file mode 100644 index 0000000..16eee19 --- /dev/null +++ b/setup/tests/token.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; + +import { evaluateDatasetWriteToken } from "../src/token.js"; + +function whoami(scoped: readonly unknown[], role = "fineGrained"): unknown { + return { + name: "owner", + auth: { + accessToken: { + displayName: "dataset-writer", + role, + fineGrained: { global: [], scoped }, + }, + }, + }; +} + +function scope(name: string, permissions: readonly string[], type = "dataset"): unknown { + return { entity: { type, name }, permissions }; +} + +describe("dataset token verification", () => { + it("accepts a fine-grained token scoped only to the dataset repo", () => { + const report = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.content.read", "repo.content.write"])]), + "alice/xtap-pool-data", + ); + expect(report).toEqual({ + ok: true, + username: "owner", + tokenName: "dataset-writer", + permissions: ["repo.content.read", "repo.content.write"], + }); + }); + + it("accepts dataset entity names with a datasets prefix", () => { + const report = evaluateDatasetWriteToken( + whoami([scope("datasets/alice/xtap-pool-data", ["repo.content.write"])]), + "alice/xtap-pool-data", + ); + expect(report.ok).toBe(true); + }); + + it("rejects classic or broad tokens", () => { + const report = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.content.write"])], "write"), + "alice/xtap-pool-data", + ); + expect(report.ok).toBe(false); + if (!report.ok) expect(report.errors[0]).toContain("fine-grained"); + }); + + it("rejects permissions outside the target dataset", () => { + const report = evaluateDatasetWriteToken( + whoami([ + scope("alice/xtap-pool-data", ["repo.content.write"]), + scope("alice/other-dataset", ["repo.content.read"]), + ]), + "alice/xtap-pool-data", + ); + expect(report.ok).toBe(false); + if (!report.ok) expect(report.errors.join("\n")).toContain("outside alice/xtap-pool-data"); + }); + + it("rejects missing write or unexpected target permissions", () => { + const missing = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.content.read"])]), + "alice/xtap-pool-data", + ); + const unexpected = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.settings.write"])]), + "alice/xtap-pool-data", + ); + expect(missing.ok).toBe(false); + expect(unexpected.ok).toBe(false); + }); +}); diff --git a/setup/tsconfig.json b/setup/tsconfig.json new file mode 100644 index 0000000..d2f4547 --- /dev/null +++ b/setup/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "dist", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "types": ["node"] + }, + "include": ["src", "tests"] +} diff --git a/setup/tsconfig.typecheck.json b/setup/tsconfig.typecheck.json new file mode 100644 index 0000000..9e6613e --- /dev/null +++ b/setup/tsconfig.typecheck.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "composite": false, + "noEmit": true, + "declaration": false, + "declarationMap": false + }, + "include": ["src", "tests", "vitest.config.ts"] +} diff --git a/setup/vitest.config.ts b/setup/vitest.config.ts new file mode 100644 index 0000000..b56adb0 --- /dev/null +++ b/setup/vitest.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + name: "setup", + include: ["tests/**/*.test.ts"], + coverage: { + provider: "v8", + include: ["src/**"], + exclude: ["src/main.ts", "src/wizard.ts", "src/deploy.ts"], + thresholds: { + lines: 85, + functions: 85, + branches: 85, + statements: 85, + }, + }, + }, +}); diff --git a/slophammer.yml b/slophammer.yml index 9d096ba..e2e897b 100644 --- a/slophammer.yml +++ b/slophammer.yml @@ -5,6 +5,7 @@ typescript: - shared/src - space/src - explorer/src + - setup/src exclude: - pattern: "explorer/src/components/ui/**" reason: "generated shadcn/ui primitives, exercised through component tests" @@ -12,6 +13,12 @@ typescript: reason: "DOM bootstrap entrypoint with no branching logic" - pattern: "space/src/server.ts" reason: "process entrypoint wiring env to app factory, covered by integration boot test" + - pattern: "setup/src/deploy.ts" + reason: "network deployment orchestration, covered through lower-level setup helpers" + - pattern: "setup/src/main.ts" + reason: "CLI entrypoint wiring" + - pattern: "setup/src/wizard.ts" + reason: "interactive Clack prompts, covered through non-interactive helper tests" - pattern: "extension/**" reason: "vendored upstream xTap extension (vanilla JS), tested via node --test" complexity: @@ -22,6 +29,7 @@ typescript: - shared/src - space/src - explorer/src + - setup/src exclude: - pattern: "explorer/src/components/ui/**" reason: "generated shadcn/ui primitives" diff --git a/vitest.config.ts b/vitest.config.ts index 545bd59..2c5f5fc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,11 +7,23 @@ export default defineConfig({ }, }, test: { - projects: ["shared/vitest.config.ts", "space/vitest.config.ts", "explorer/vitest.config.ts"], + projects: [ + "shared/vitest.config.ts", + "space/vitest.config.ts", + "explorer/vitest.config.ts", + "setup/vitest.config.ts", + ], coverage: { provider: "v8", - include: ["shared/src/**", "space/src/**", "explorer/src/**"], - exclude: ["explorer/src/components/ui/**", "explorer/src/main.tsx", "space/src/server.ts"], + include: ["shared/src/**", "space/src/**", "explorer/src/**", "setup/src/**"], + exclude: [ + "explorer/src/components/ui/**", + "explorer/src/main.tsx", + "setup/src/deploy.ts", + "setup/src/main.ts", + "setup/src/wizard.ts", + "space/src/server.ts", + ], thresholds: { lines: 85, functions: 85, From 20973074706fbfc7f2502cbea75292e16cad3040 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:39:09 +0200 Subject: [PATCH 02/11] fix: run setup wizard from compiled entrypoint --- package.json | 2 +- setup/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ebce5d3..e2c0f05 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "setup" ], "scripts": { - "setup": "npm run dev --workspace setup", + "setup": "npm run build --workspace setup && npm run start --workspace setup", "format": "prettier --check .", "format:write": "prettier --write .", "lint": "eslint .", diff --git a/setup/package.json b/setup/package.json index e408b5e..ca5c6c6 100644 --- a/setup/package.json +++ b/setup/package.json @@ -8,7 +8,7 @@ }, "scripts": { "build": "tsc --build", - "dev": "node --experimental-strip-types src/main.ts", + "dev": "npm run build && npm run start", "start": "node dist/src/main.js", "format": "prettier --check .", "lint": "eslint .", From 5d92fae1d16858b0332b107531e6da3b2386d296 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:45:24 +0200 Subject: [PATCH 03/11] fix: keep setup manifest in space build context --- Dockerfile | 2 ++ setup/src/stage.ts | 4 ++++ setup/tests/stage.test.ts | 9 ++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 716dcfc..9d9c22e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ COPY package.json package-lock.json ./ COPY shared/package.json shared/ COPY space/package.json space/ COPY explorer/package.json explorer/ +COPY setup/package.json setup/ RUN npm ci COPY tsconfig.base.json ./ COPY shared/ shared/ @@ -20,6 +21,7 @@ COPY package.json package-lock.json ./ COPY shared/package.json shared/ COPY space/package.json space/ COPY explorer/package.json explorer/ +COPY setup/package.json setup/ RUN npm ci --omit=dev && npm cache clean --force COPY --from=build /app/shared/dist shared/dist COPY --from=build /app/space/dist space/dist diff --git a/setup/src/stage.ts b/setup/src/stage.ts index 902bac4..43c9d40 100644 --- a/setup/src/stage.ts +++ b/setup/src/stage.ts @@ -10,6 +10,7 @@ export type UploadFile = { }; const SPACE_EXCLUDED_ROOTS = new Set(["docs", "extension", "setup"]); +const SPACE_ALLOWED_FILES = new Set(["setup/package.json"]); export async function createSpaceStage(root: string, stageDir: string): Promise { const archivePath = join(stageDir, "repo.tar"); @@ -22,6 +23,8 @@ export async function createSpaceStage(root: string, stageDir: string): Promise< fs.rm(join(stageDir, name), { recursive: true, force: true }), ), ); + await fs.mkdir(join(stageDir, "setup"), { recursive: true }); + await fs.copyFile(join(root, "setup", "package.json"), join(stageDir, "setup", "package.json")); } export async function collectUploadFiles(root: string): Promise { @@ -35,6 +38,7 @@ export async function collectUploadFiles(root: string): Promise { expect(shouldUploadPath("space/src/server.ts")).toBe(true); expect(shouldUploadPath("docs/implementation-plan.md")).toBe(false); expect(shouldUploadPath("extension/manifest.json")).toBe(false); + expect(shouldUploadPath("setup/package.json")).toBe(true); expect(shouldUploadPath("setup/src/main.ts")).toBe(false); }); @@ -42,13 +43,14 @@ describe("space staging helpers", () => { await mkdir(join(repo, "space"), { recursive: true }); await mkdir(join(repo, "docs"), { recursive: true }); await mkdir(join(repo, "extension"), { recursive: true }); - await mkdir(join(repo, "setup"), { recursive: true }); + await mkdir(join(repo, "setup", "src"), { recursive: true }); await writeFile(join(repo, "README.md"), "root readme"); await writeFile(join(repo, "space", "hf-space-README.md"), "space readme"); await writeFile(join(repo, "space", "server.ts"), "export {};"); await writeFile(join(repo, "docs", "plan.md"), "internal"); await writeFile(join(repo, "extension", "manifest.json"), "{}"); - await writeFile(join(repo, "setup", "main.ts"), "export {};"); + await writeFile(join(repo, "setup", "package.json"), "{}"); + await writeFile(join(repo, "setup", "src", "main.ts"), "export {};"); await captureCommand("git", ["add", "."], { cwd: repo }); await captureCommand( "git", @@ -62,9 +64,10 @@ describe("space staging helpers", () => { await expect(readFile(join(stage, "README.md"), "utf8")).resolves.toBe("space readme"); await expect(readFile(join(stage, "space", "server.ts"), "utf8")).resolves.toBe("export {};"); + await expect(readFile(join(stage, "setup", "package.json"), "utf8")).resolves.toBe("{}"); await expect(readdir(join(stage, "docs"))).rejects.toThrow(); await expect(readdir(join(stage, "extension"))).rejects.toThrow(); - await expect(readdir(join(stage, "setup"))).rejects.toThrow(); + await expect(readdir(join(stage, "setup", "src"))).rejects.toThrow(); } finally { await rm(repo, { recursive: true, force: true }); await rm(stage, { recursive: true, force: true }); From 7970ca24a577517156822de0dac84e3333858ed0 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:53:21 +0200 Subject: [PATCH 04/11] fix: handle empty Space secret responses --- setup/src/hub-api.ts | 19 +++++++++++++++---- setup/tests/hub-api.test.ts | 3 +++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/setup/src/hub-api.ts b/setup/src/hub-api.ts index 28bba3d..92c44e6 100644 --- a/setup/src/hub-api.ts +++ b/setup/src/hub-api.ts @@ -12,7 +12,9 @@ export async function getSpaceVariables( client: HubClient, spaceRepo: string, ): Promise> { - const payload = await hubJson(client, `/api/spaces/${spaceRepo}/variables`, { method: "GET" }); + const payload = await hubRequestJson(client, `/api/spaces/${spaceRepo}/variables`, { + method: "GET", + }); return parseSpaceVariables(payload); } @@ -22,7 +24,7 @@ export async function setSpaceVariable( key: string, value: string, ): Promise { - await hubJson(client, `/api/spaces/${spaceRepo}/variables`, { + await hubRequestJson(client, `/api/spaces/${spaceRepo}/variables`, { method: "POST", body: JSON.stringify({ key, value }), }); @@ -34,7 +36,7 @@ export async function setSpaceSecret( key: string, value: string, ): Promise { - await hubJson(client, `/api/spaces/${spaceRepo}/secrets`, { + await hubRequest(client, `/api/spaces/${spaceRepo}/secrets`, { method: "POST", body: JSON.stringify({ key, value }), }); @@ -50,7 +52,7 @@ export function parseSpaceVariables(payload: unknown): ReadonlyMap { +async function hubRequest(client: HubClient, path: string, init: RequestInit): Promise { const headers = new Headers(init.headers); headers.set("authorization", `Bearer ${client.accessToken}`); headers.set("content-type", "application/json"); @@ -61,6 +63,15 @@ async function hubJson(client: HubClient, path: string, init: RequestInit): Prom if (!response.ok) { throw new Error(`Hub request failed (${String(response.status)}): ${await response.text()}`); } + return response; +} + +async function hubRequestJson( + client: HubClient, + path: string, + init: RequestInit, +): Promise { + const response = await hubRequest(client, path, init); const payload: unknown = await response.json(); return asRecord(payload); } diff --git a/setup/tests/hub-api.test.ts b/setup/tests/hub-api.test.ts index c954fde..0349145 100644 --- a/setup/tests/hub-api.test.ts +++ b/setup/tests/hub-api.test.ts @@ -24,6 +24,9 @@ describe("space variable parsing", () => { const requests: { url: string; init: RequestInit }[] = []; const fetchFn: typeof fetch = (input, init) => { requests.push({ url: requestUrl(input), init: init ?? {} }); + if (requestUrl(input).endsWith("/secrets")) { + return Promise.resolve(new Response(null, { status: 204 })); + } return Promise.resolve(Response.json({ DATASET_REPO: { value: "alice/xtap-pool-data" } })); }; const client = { accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn }; From 62f1c7d0c39abc80c14a26234e23675b981ca6f9 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:18:53 +0200 Subject: [PATCH 05/11] fix: derive setup repo defaults from namespace --- setup/src/config.ts | 4 ++++ setup/src/wizard.ts | 16 +++++++++++++--- setup/tests/config.test.ts | 3 +++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/setup/src/config.ts b/setup/src/config.ts index 48ab7da..296a584 100644 --- a/setup/src/config.ts +++ b/setup/src/config.ts @@ -40,6 +40,10 @@ export function validateRepoId(value: string): string | undefined { return REPO_ID.test(value) ? undefined : "Use owner/name, for example osolmaz/xtap-pool."; } +export function validateNamespace(value: string): string | undefined { + return USERNAME.test(value) ? undefined : "Use a Hugging Face username or organization name."; +} + export function validateUserList(value: string): string | undefined { const users = normalizeUsers(value); if (users.length === 0) return "Enter at least one Hugging Face username."; diff --git a/setup/src/wizard.ts b/setup/src/wizard.ts index 7e61096..e203afc 100644 --- a/setup/src/wizard.ts +++ b/setup/src/wizard.ts @@ -15,9 +15,11 @@ import type { SetupConfig } from "./config.js"; import { defaultSetupConfig, normalizeUsers, + repoInNamespace, spacePublicUrl, tokenSettingsUrl, usersValue, + validateNamespace, validateRepoId, validateUserList, } from "./config.js"; @@ -51,11 +53,19 @@ async function activeHfToken(): Promise { async function promptConfig(username: string): Promise { const defaults = defaultSetupConfig(username); - const namespace = await promptText("Hugging Face namespace", defaults.namespace); - const spaceRepo = await promptText("Space repo", defaults.spaceRepo, validateRepoId); + const namespace = await promptText( + "Hugging Face namespace", + defaults.namespace, + validateNamespace, + ); + const spaceRepo = await promptText( + "Space repo", + repoInNamespace(namespace, "xtap-pool"), + validateRepoId, + ); const datasetRepo = await promptText( "Private dataset repo", - defaults.datasetRepo, + repoInNamespace(namespace, "xtap-pool-data"), validateRepoId, ); const allowed = await promptText( diff --git a/setup/tests/config.test.ts b/setup/tests/config.test.ts index ea7863a..b8ef559 100644 --- a/setup/tests/config.test.ts +++ b/setup/tests/config.test.ts @@ -7,6 +7,7 @@ import { spacePublicUrl, tokenSettingsUrl, usersValue, + validateNamespace, validateRepoId, validateUserList, } from "../src/config.js"; @@ -27,6 +28,8 @@ describe("setup config helpers", () => { }); it("validates repo ids and user lists", () => { + expect(validateNamespace("dutifuldev")).toBeUndefined(); + expect(validateNamespace("bad namespace")).toContain("username or organization"); expect(validateRepoId("alice/xtap-pool")).toBeUndefined(); expect(validateRepoId("xtap-pool")).toContain("owner/name"); expect(validateUserList("alice,bob")).toBeUndefined(); From 450bf26ce947733b081737e8efd16ea136914122 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:26:09 +0200 Subject: [PATCH 06/11] fix: require dataset token read access --- setup/src/token.ts | 8 +++++--- setup/src/wizard.ts | 3 ++- setup/tests/token.test.ts | 13 +++++++++---- space/hf-space-README.md | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/setup/src/token.ts b/setup/src/token.ts index 2c4ea59..e3379c3 100644 --- a/setup/src/token.ts +++ b/setup/src/token.ts @@ -2,7 +2,7 @@ type JsonObject = Record; const FINE_GRAINED_ROLE = "fineGrained"; const TARGET_PERMISSIONS = new Set(["repo.content.read", "repo.content.write"]); -const REQUIRED_WRITE_PERMISSION = "repo.content.write"; +const REQUIRED_PERMISSIONS = ["repo.content.read", "repo.content.write"] as const; export type DatasetTokenReport = | { @@ -44,8 +44,10 @@ export function evaluateDatasetWriteToken( : [`Token role is '${role || "unknown"}', expected fine-grained.`]; errors.push(...globalPermissionErrors(fineGrained)); const targetPermissions = scopedPermissionErrors(fineGrained, datasetRepo, errors); - if (!targetPermissions.has(REQUIRED_WRITE_PERMISSION)) { - errors.push(`Token must include ${REQUIRED_WRITE_PERMISSION} on ${datasetRepo}.`); + for (const permission of REQUIRED_PERMISSIONS) { + if (!targetPermissions.has(permission)) { + errors.push(`Token must include ${permission} on ${datasetRepo}.`); + } } if (errors.length > 0) return { ok: false, errors }; return { diff --git a/setup/src/wizard.ts b/setup/src/wizard.ts index e203afc..658ac17 100644 --- a/setup/src/wizard.ts +++ b/setup/src/wizard.ts @@ -100,7 +100,8 @@ async function confirmPlan(config: SetupConfig): Promise { async function promptDatasetToken(datasetRepo: string): Promise { note( [ - `Create a fine-grained token with write access only to ${datasetRepo}.`, + `Create a fine-grained token scoped only to ${datasetRepo}.`, + `It must include repo.content.read and repo.content.write on that dataset.`, tokenSettingsUrl(), ].join("\n"), "Dataset token", diff --git a/setup/tests/token.test.ts b/setup/tests/token.test.ts index 16eee19..e0e8653 100644 --- a/setup/tests/token.test.ts +++ b/setup/tests/token.test.ts @@ -35,7 +35,7 @@ describe("dataset token verification", () => { it("accepts dataset entity names with a datasets prefix", () => { const report = evaluateDatasetWriteToken( - whoami([scope("datasets/alice/xtap-pool-data", ["repo.content.write"])]), + whoami([scope("datasets/alice/xtap-pool-data", ["repo.content.read", "repo.content.write"])]), "alice/xtap-pool-data", ); expect(report.ok).toBe(true); @@ -62,16 +62,21 @@ describe("dataset token verification", () => { if (!report.ok) expect(report.errors.join("\n")).toContain("outside alice/xtap-pool-data"); }); - it("rejects missing write or unexpected target permissions", () => { - const missing = evaluateDatasetWriteToken( + it("rejects missing read, missing write, or unexpected target permissions", () => { + const missingWrite = evaluateDatasetWriteToken( whoami([scope("alice/xtap-pool-data", ["repo.content.read"])]), "alice/xtap-pool-data", ); + const missingRead = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.content.write"])]), + "alice/xtap-pool-data", + ); const unexpected = evaluateDatasetWriteToken( whoami([scope("alice/xtap-pool-data", ["repo.settings.write"])]), "alice/xtap-pool-data", ); - expect(missing.ok).toBe(false); + expect(missingRead.ok).toBe(false); + expect(missingWrite.ok).toBe(false); expect(unexpected.ok).toBe(false); }); }); diff --git a/space/hf-space-README.md b/space/hf-space-README.md index c7283fb..64c9a1d 100644 --- a/space/hf-space-README.md +++ b/space/hf-space-README.md @@ -19,7 +19,7 @@ Private tweet pool for a group of friends running the - `GET /connect` — sign in with Hugging Face to connect the extension - `/` — tweet explorer -Required Space secrets: `HF_TOKEN` (fine-grained, write access to the dataset -repo only), `POOL_SIGNING_SECRET`, `SESSION_SECRET`. +Required Space secrets: `HF_TOKEN` (fine-grained, read/write access to the +dataset repo only), `POOL_SIGNING_SECRET`, `SESSION_SECRET`. Required Space variables: `DATASET_REPO`, `ALLOWED_USERS` (comma-separated HF usernames), `SPACE_HOST` (auto-injected by HF). From 60b701598245efe7e653b2b15600d08a78befb52 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:29:26 +0200 Subject: [PATCH 07/11] docs: clarify dataset token permissions --- docs/implementation-plan.md | 4 ++-- scripts/deploy-space.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index ce5041c..0efa78c 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -178,8 +178,8 @@ buffering; ephemeral disk must never hold unpersisted data). Commits use `@huggingface/hub` `commit()` with retry; concurrent ingests serialized through a single writer queue. -Config via Space secrets/variables: `HF_TOKEN` (fine-grained, write on the -one dataset repo), `POOL_SIGNING_SECRET`, `SESSION_SECRET`, `ALLOWED_USERS` +Config via Space secrets/variables: `HF_TOKEN` (fine-grained, read/write on +the one dataset repo), `POOL_SIGNING_SECRET`, `SESSION_SECRET`, `ALLOWED_USERS` (comma-separated HF usernames), `DATASET_REPO`. README metadata: `sdk: docker`, `hf_oauth: true`, scopes `openid profile`. diff --git a/scripts/deploy-space.sh b/scripts/deploy-space.sh index acbe4a0..8698390 100755 --- a/scripts/deploy-space.sh +++ b/scripts/deploy-space.sh @@ -56,7 +56,7 @@ if "SECRETS_INITIALIZED" not in variables: api.add_space_variable(space, "SECRETS_INITIALIZED", "1") print("Set DATASET_REPO, ALLOWED_USERS, POOL_SIGNING_SECRET, SESSION_SECRET.") print("Remaining manual steps:") -print(f" 1. Create a fine-grained token with write access to {dataset} only,") +print(f" 1. Create a fine-grained token with read/write access to {dataset} only,") print(f" then: python3 -c \"from huggingface_hub import HfApi; HfApi().add_space_secret('{space}', 'HF_TOKEN', '')\"") print(f" 2. Optionally seed history: scripts/seed-dataset.sh {dataset} ~/xtap-store/data/tweets") PY From a17cfccf5098b3fa495ee0d478487d160087976a Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:46:28 +0200 Subject: [PATCH 08/11] test: add setup mutation gate --- .github/workflows/ci.yml | 1 + setup/package.json | 3 ++- setup/stryker.conf.json | 14 ++++++++++++++ stryker.conf.json | 9 ++++++++- 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 setup/stryker.conf.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b7b44b..e3c23e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: npm run test --workspace setup npm run coverage --workspace setup npm run dry --workspace setup + npm run mutate --workspace setup - run: npm run build --workspace explorer && npm run build --workspace space && npm run build --workspace setup - name: Slophammer uses: dutifuldev/slophammer@main diff --git a/setup/package.json b/setup/package.json index ca5c6c6..777a6a1 100644 --- a/setup/package.json +++ b/setup/package.json @@ -15,7 +15,8 @@ "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", "test": "vitest run", "coverage": "vitest run --coverage", - "dry": "cd .. && slophammer-ts dry ." + "dry": "cd .. && slophammer-ts dry .", + "mutate": "stryker run stryker.conf.json" }, "dependencies": { "@clack/prompts": "^0.11.0", diff --git a/setup/stryker.conf.json b/setup/stryker.conf.json new file mode 100644 index 0000000..3fcf455 --- /dev/null +++ b/setup/stryker.conf.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker-js/master/packages/api/schema/stryker-core.json", + "testRunner": "vitest", + "mutate": ["src/config.ts", "src/hub-api.ts", "src/token.ts"], + "vitest": { + "configFile": "vitest.config.ts" + }, + "thresholds": { + "high": 80, + "low": 60, + "break": 60 + }, + "reporters": ["clear-text", "progress"] +} diff --git a/stryker.conf.json b/stryker.conf.json index fd13f9c..fc6e9fb 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -1,7 +1,14 @@ { "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker-js/master/packages/api/schema/stryker-core.json", "testRunner": "vitest", - "mutate": ["shared/src/tweet.ts", "space/src/pool-token.ts", "explorer/src/lib/format.ts"], + "mutate": [ + "shared/src/tweet.ts", + "space/src/pool-token.ts", + "explorer/src/lib/format.ts", + "setup/src/config.ts", + "setup/src/hub-api.ts", + "setup/src/token.ts" + ], "vitest": { "configFile": "vitest.config.ts" }, From 1b2506224fef7de11b4a7980a0cfd63555023cec Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:56:33 +0200 Subject: [PATCH 09/11] fix: harden xtap pool setup flow --- setup/.prettierignore | 5 +++++ setup/src/deploy.ts | 29 +++++++++++++++++++++++++---- setup/src/hub-api.ts | 14 +++++++++++++- setup/src/path.ts | 12 ++++++++++++ setup/src/token.ts | 10 +++++++++- setup/src/wizard.ts | 11 +++++++---- setup/tests/hub-api.test.ts | 13 ++++++++++++- setup/tests/path.test.ts | 13 +++++++++++++ setup/tests/token.test.ts | 9 +++++++++ 9 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 setup/.prettierignore create mode 100644 setup/src/path.ts create mode 100644 setup/tests/path.test.ts diff --git a/setup/.prettierignore b/setup/.prettierignore new file mode 100644 index 0000000..fe1d0b8 --- /dev/null +++ b/setup/.prettierignore @@ -0,0 +1,5 @@ +coverage/ +dist/ +.stryker-tmp/ +reports/ +*.tsbuildinfo diff --git a/setup/src/deploy.ts b/setup/src/deploy.ts index 2ecc10c..d786fce 100644 --- a/setup/src/deploy.ts +++ b/setup/src/deploy.ts @@ -7,8 +7,13 @@ import { createRepo, repoExists, uploadFiles } from "@huggingface/hub"; import type { SetupConfig } from "./config.js"; import { usersValue } from "./config.js"; -import type { HubClient } from "./hub-api.js"; -import { getSpaceVariables, setSpaceSecret, setSpaceVariable } from "./hub-api.js"; +import type { HubClient, HubRepo } from "./hub-api.js"; +import { + getRepoPrivateState, + getSpaceVariables, + setSpaceSecret, + setSpaceVariable, +} from "./hub-api.js"; import { captureCommand } from "./process.js"; import { collectUploadFiles, createSpaceStage } from "./stage.js"; @@ -41,7 +46,7 @@ export async function configureSpace(client: HubClient, config: SetupConfig): Pr async function ensureRepo( client: HubClient, - repo: { type: "dataset" | "space"; name: string }, + repo: HubRepo, visibility: "private" | "public", ): Promise { const exists = await repoExists({ @@ -49,7 +54,10 @@ async function ensureRepo( accessToken: client.accessToken, ...fetchOption(client), }); - if (exists) return; + if (exists) { + await assertRepoVisibility(client, repo, visibility); + return; + } await createRepo({ repo, accessToken: client.accessToken, @@ -59,6 +67,19 @@ async function ensureRepo( }); } +async function assertRepoVisibility( + client: HubClient, + repo: HubRepo, + visibility: "private" | "public", +): Promise { + const actualPrivate = await getRepoPrivateState(client, repo); + const expectedPrivate = visibility === "private"; + if (actualPrivate === expectedPrivate) return; + throw new Error( + `${repo.type} ${repo.name} already exists as ${actualPrivate ? "private" : "public"}; expected ${visibility}.`, + ); +} + async function uploadSpace(root: string, client: HubClient, spaceRepo: string): Promise { const stageDir = await mkdtemp(join(tmpdir(), "xtap-pool-space-")); try { diff --git a/setup/src/hub-api.ts b/setup/src/hub-api.ts index 92c44e6..413c2fa 100644 --- a/setup/src/hub-api.ts +++ b/setup/src/hub-api.ts @@ -6,6 +6,8 @@ export type HubClient = { fetchFn?: typeof fetch; }; +export type HubRepo = { type: "dataset" | "space"; name: string }; + type JsonObject = Record; export async function getSpaceVariables( @@ -24,7 +26,7 @@ export async function setSpaceVariable( key: string, value: string, ): Promise { - await hubRequestJson(client, `/api/spaces/${spaceRepo}/variables`, { + await hubRequest(client, `/api/spaces/${spaceRepo}/variables`, { method: "POST", body: JSON.stringify({ key, value }), }); @@ -42,6 +44,16 @@ export async function setSpaceSecret( }); } +export async function getRepoPrivateState(client: HubClient, repo: HubRepo): Promise { + const payload = await hubRequestJson(client, `/api/${repo.type}s/${repo.name}`, { + method: "GET", + }); + if (typeof payload["private"] !== "boolean") { + throw new Error(`Hub ${repo.type} ${repo.name} did not report visibility.`); + } + return payload["private"]; +} + export function parseSpaceVariables(payload: unknown): ReadonlyMap { const result = new Map(); for (const [key, value] of Object.entries(asRecord(payload))) { diff --git a/setup/src/path.ts b/setup/src/path.ts new file mode 100644 index 0000000..e70c9c3 --- /dev/null +++ b/setup/src/path.ts @@ -0,0 +1,12 @@ +import { homedir } from "node:os"; +import { join } from "node:path"; + +export function defaultTweetsDirectory(home = homedir()): string { + return join(home, "xtap-store/data/tweets"); +} + +export function expandHomePath(value: string, home = homedir()): string { + if (value === "~") return home; + if (value.startsWith("~/")) return join(home, value.slice(2)); + return value; +} diff --git a/setup/src/token.ts b/setup/src/token.ts index e3379c3..33ed790 100644 --- a/setup/src/token.ts +++ b/setup/src/token.ts @@ -104,7 +104,15 @@ function recordTargetPermission( } function matchesDataset(entity: JsonObject, datasetRepo: string): boolean { - return entityCandidates(entity).some((candidate) => normalizeRepo(candidate) === datasetRepo); + return ( + isDatasetEntity(entity) && + entityCandidates(entity).some((candidate) => normalizeRepo(candidate) === datasetRepo) + ); +} + +function isDatasetEntity(entity: JsonObject): boolean { + const type = text(entity["type"]); + return type === "dataset" || type === "datasets"; } function entityCandidates(entity: JsonObject): readonly string[] { diff --git a/setup/src/wizard.ts b/setup/src/wizard.ts index 658ac17..f657da2 100644 --- a/setup/src/wizard.ts +++ b/setup/src/wizard.ts @@ -25,6 +25,7 @@ import { } from "./config.js"; import { deployPool } from "./deploy.js"; import { setSpaceSecret } from "./hub-api.js"; +import { defaultTweetsDirectory, expandHomePath } from "./path.js"; import { captureCommand, inheritCommand } from "./process.js"; import { verifyDatasetWriteToken } from "./token.js"; @@ -127,10 +128,12 @@ async function maybeSeed(root: string, config: SetupConfig): Promise { "Seed data belongs to which HF user?", config.allowedUsers[0] ?? config.namespace, ); - const source = await promptText("Existing tweets directory", "~/xtap-store/data/tweets"); - await inheritCommand("scripts/seed-dataset.sh", [config.datasetRepo, username, source], { - cwd: root, - }); + const source = await promptText("Existing tweets directory", defaultTweetsDirectory()); + await inheritCommand( + "scripts/seed-dataset.sh", + [config.datasetRepo, username, expandHomePath(source)], + { cwd: root }, + ); } async function promptText( diff --git a/setup/tests/hub-api.test.ts b/setup/tests/hub-api.test.ts index 0349145..ffcb81c 100644 --- a/setup/tests/hub-api.test.ts +++ b/setup/tests/hub-api.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { + getRepoPrivateState, getSpaceVariables, parseSpaceVariables, setSpaceSecret, @@ -24,7 +25,7 @@ describe("space variable parsing", () => { const requests: { url: string; init: RequestInit }[] = []; const fetchFn: typeof fetch = (input, init) => { requests.push({ url: requestUrl(input), init: init ?? {} }); - if (requestUrl(input).endsWith("/secrets")) { + if (init?.method === "POST") { return Promise.resolve(new Response(null, { status: 204 })); } return Promise.resolve(Response.json({ DATASET_REPO: { value: "alice/xtap-pool-data" } })); @@ -47,6 +48,16 @@ describe("space variable parsing", () => { expect((headers as Headers).get("authorization")).toBe("Bearer hf_owner"); }); + it("reads repo visibility from Hub info", async () => { + const fetchFn: typeof fetch = () => Promise.resolve(Response.json({ private: true })); + await expect( + getRepoPrivateState( + { accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn }, + { type: "dataset", name: "alice/xtap-pool-data" }, + ), + ).resolves.toBe(true); + }); + it("surfaces Hub errors with status and body", async () => { const fetchFn: typeof fetch = () => Promise.resolve(new Response("bad token", { status: 401 })); await expect( diff --git a/setup/tests/path.test.ts b/setup/tests/path.test.ts new file mode 100644 index 0000000..5dc6c84 --- /dev/null +++ b/setup/tests/path.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; + +import { expandHomePath } from "../src/path.js"; + +describe("setup path helpers", () => { + it("expands leading home paths", () => { + expect(expandHomePath("~", "/home/alice")).toBe("/home/alice"); + expect(expandHomePath("~/xtap-store/data/tweets", "/home/alice")).toBe( + "/home/alice/xtap-store/data/tweets", + ); + expect(expandHomePath("/tmp/tweets", "/home/alice")).toBe("/tmp/tweets"); + }); +}); diff --git a/setup/tests/token.test.ts b/setup/tests/token.test.ts index e0e8653..5c3a0c8 100644 --- a/setup/tests/token.test.ts +++ b/setup/tests/token.test.ts @@ -62,6 +62,15 @@ describe("dataset token verification", () => { if (!report.ok) expect(report.errors.join("\n")).toContain("outside alice/xtap-pool-data"); }); + it("rejects same-name scopes on non-dataset entities", () => { + const report = evaluateDatasetWriteToken( + whoami([scope("alice/xtap-pool-data", ["repo.content.read", "repo.content.write"], "model")]), + "alice/xtap-pool-data", + ); + expect(report.ok).toBe(false); + if (!report.ok) expect(report.errors.join("\n")).toContain("model:alice/xtap-pool-data"); + }); + it("rejects missing read, missing write, or unexpected target permissions", () => { const missingWrite = evaluateDatasetWriteToken( whoami([scope("alice/xtap-pool-data", ["repo.content.read"])]), From 85901bdc2388719904dd93654345393debec82ad Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 02:10:38 +0200 Subject: [PATCH 10/11] fix: delete stale space files during setup deploy --- setup/src/deploy.ts | 56 ++++++++++++++++++++++++++++++++------ setup/tests/deploy.test.ts | 38 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 setup/tests/deploy.test.ts diff --git a/setup/src/deploy.ts b/setup/src/deploy.ts index d786fce..f7880d1 100644 --- a/setup/src/deploy.ts +++ b/setup/src/deploy.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createRepo, repoExists, uploadFiles } from "@huggingface/hub"; +import { commit, createRepo, listFiles, repoExists } from "@huggingface/hub"; import type { SetupConfig } from "./config.js"; import { usersValue } from "./config.js"; @@ -17,6 +17,8 @@ import { import { captureCommand } from "./process.js"; import { collectUploadFiles, createSpaceStage } from "./stage.js"; +type DeleteOperation = { operation: "delete"; path: string }; + export async function deployPool( root: string, client: HubClient, @@ -52,7 +54,7 @@ async function ensureRepo( const exists = await repoExists({ repo, accessToken: client.accessToken, - ...fetchOption(client), + ...hubOptions(client), }); if (exists) { await assertRepoVisibility(client, repo, visibility); @@ -63,7 +65,7 @@ async function ensureRepo( accessToken: client.accessToken, visibility, ...(repo.type === "space" ? { sdk: "docker" as const } : {}), - ...fetchOption(client), + ...hubOptions(client), }); } @@ -84,18 +86,51 @@ async function uploadSpace(root: string, client: HubClient, spaceRepo: string): const stageDir = await mkdtemp(join(tmpdir(), "xtap-pool-space-")); try { await createSpaceStage(root, stageDir); - await uploadFiles({ + const files = await collectUploadFiles(stageDir); + const staleDeletes = await collectStaleSpaceDeletes( + client, + spaceRepo, + files.map((file) => file.path), + ); + await commit({ repo: { type: "space", name: spaceRepo }, accessToken: client.accessToken, - files: [...(await collectUploadFiles(stageDir))], - commitTitle: `deploy: ${await shortHead(root)}`, - ...fetchOption(client), + title: `deploy: ${await shortHead(root)}`, + operations: [ + ...files.map((file) => ({ + operation: "addOrUpdate" as const, + path: file.path, + content: file.content, + })), + ...staleDeletes, + ], + ...hubOptions(client), }); } finally { await rm(stageDir, { recursive: true, force: true }); } } +export async function collectStaleSpaceDeletes( + client: HubClient, + spaceRepo: string, + stagedPaths: readonly string[], +): Promise { + const desired = new Set(stagedPaths); + const deletes: DeleteOperation[] = []; + for await (const entry of listFiles({ + repo: { type: "space", name: spaceRepo }, + accessToken: client.accessToken, + recursive: true, + ...hubOptions(client), + })) { + if (entry.type === "file" && entry.path !== ".gitattributes" && !desired.has(entry.path)) { + deletes.push({ operation: "delete", path: entry.path }); + } + } + return deletes; +} + async function shortHead(root: string): Promise { const result = await captureCommand("git", ["-C", root, "rev-parse", "--short", "HEAD"]); return result.stdout.trim(); @@ -105,6 +140,9 @@ function randomSecret(): string { return randomBytes(32).toString("hex"); } -function fetchOption(client: HubClient): { fetch?: typeof fetch } { - return client.fetchFn === undefined ? {} : { fetch: client.fetchFn }; +function hubOptions(client: HubClient): { fetch?: typeof fetch; hubUrl?: string } { + return { + ...(client.fetchFn === undefined ? {} : { fetch: client.fetchFn }), + ...(client.hubUrl === undefined ? {} : { hubUrl: client.hubUrl }), + }; } diff --git a/setup/tests/deploy.test.ts b/setup/tests/deploy.test.ts new file mode 100644 index 0000000..f7096b9 --- /dev/null +++ b/setup/tests/deploy.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; + +import { collectStaleSpaceDeletes } from "../src/deploy.js"; + +describe("setup deployment helpers", () => { + it("builds delete operations for remote Space files missing from the staged upload", async () => { + const requests: { url: string; init: RequestInit }[] = []; + const fetchFn: typeof fetch = (input, init) => { + requests.push({ url: requestUrl(input), init: init ?? {} }); + return Promise.resolve( + Response.json([ + { type: "file", path: "README.md", size: 1 }, + { type: "file", path: "space/old.ts", size: 1 }, + { type: "directory", path: "space", size: 0 }, + { type: "file", path: ".gitattributes", size: 1 }, + ]), + ); + }; + + await expect( + collectStaleSpaceDeletes( + { accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn }, + "alice/xtap-pool", + ["README.md", "Dockerfile"], + ), + ).resolves.toEqual([{ operation: "delete", path: "space/old.ts" }]); + expect(requests[0]?.url).toBe( + "https://hub.test/api/spaces/alice/xtap-pool/tree/main?recursive=true&expand=false", + ); + expect(new Headers(requests[0]?.init.headers).get("authorization")).toBe("Bearer hf_owner"); + }); +}); + +function requestUrl(input: string | URL | Request): string { + if (typeof input === "string") return input; + if (input instanceof URL) return input.toString(); + return input.url; +} From 8c27ffb3846e10529c9ae29bdcc31b65fd0ca153 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 6 Jul 2026 02:25:09 +0200 Subject: [PATCH 11/11] fix: stabilize setup deployment flow --- setup/src/stage.ts | 3 ++- setup/src/wizard.ts | 2 +- setup/tests/stage.test.ts | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/src/stage.ts b/setup/src/stage.ts index 43c9d40..61e3dbe 100644 --- a/setup/src/stage.ts +++ b/setup/src/stage.ts @@ -17,6 +17,7 @@ export async function createSpaceStage(root: string, stageDir: string): Promise< await captureCommand("git", ["-C", root, "archive", "--format=tar", "-o", archivePath, "HEAD"]); await inheritCommand("tar", ["-xf", archivePath, "-C", stageDir]); await fs.rm(archivePath, { force: true }); + const setupPackageJson = await fs.readFile(join(stageDir, "setup", "package.json")); await fs.copyFile(join(root, "space", "hf-space-README.md"), join(stageDir, "README.md")); await Promise.all( [...SPACE_EXCLUDED_ROOTS].map((name) => @@ -24,7 +25,7 @@ export async function createSpaceStage(root: string, stageDir: string): Promise< ), ); await fs.mkdir(join(stageDir, "setup"), { recursive: true }); - await fs.copyFile(join(root, "setup", "package.json"), join(stageDir, "setup", "package.json")); + await fs.writeFile(join(stageDir, "setup", "package.json"), setupPackageJson); } export async function collectUploadFiles(root: string): Promise { diff --git a/setup/src/wizard.ts b/setup/src/wizard.ts index f657da2..4de9cff 100644 --- a/setup/src/wizard.ts +++ b/setup/src/wizard.ts @@ -40,8 +40,8 @@ export async function runSetupWizard(root: string): Promise { await deployPool(root, { accessToken }, config); task.stop("Space deployed"); const datasetToken = await promptDatasetToken(config.datasetRepo); - await setSpaceSecret({ accessToken }, config.spaceRepo, "HF_TOKEN", datasetToken); await maybeSeed(root, config); + await setSpaceSecret({ accessToken }, config.spaceRepo, "HF_TOKEN", datasetToken); outro(`Done. Explorer: ${spacePublicUrl(config.spaceRepo)}`); } diff --git a/setup/tests/stage.test.ts b/setup/tests/stage.test.ts index 8bb94bf..34dde4e 100644 --- a/setup/tests/stage.test.ts +++ b/setup/tests/stage.test.ts @@ -59,6 +59,7 @@ describe("space staging helpers", () => { cwd: repo, }, ); + await writeFile(join(repo, "setup", "package.json"), '{"dirty":true}'); await createSpaceStage(repo, stage);