Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,26 @@ The setup flow creates or updates the private dataset repo and public Docker
Space, configures the Space variables and generated secrets, verifies the
dataset-only `HF_TOKEN`, and can import existing xTap JSONL files.

To redeploy an existing pool without re-entering repo names, the dataset token,
or import settings:

```sh
npm run update
```

By default this updates `<active-hf-user>/xtap-pool`. Pass a Space repo when
updating a different namespace:

```sh
npm run update -- dutifuldev/xtap-pool
```

The update command reads the current Space variables, reuses the existing
dataset repo and membership bootstrap settings, preserves all secrets, and only
uploads the latest Space code plus any missing variables.
It will not create or rotate generated signing/session secrets; run the setup
flow if those were never initialized.

The lower-level scripts are still available when you want to do those steps
manually:

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
],
"scripts": {
"setup": "npm run build --workspace setup && npm run start --workspace setup",
"update": "npm run build --workspace setup && npm run start --workspace setup -- update",
"format": "prettier --check .",
"format:write": "prettier --write .",
"lint": "eslint .",
Expand Down
16 changes: 16 additions & 0 deletions setup/src/cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import { validateRepoId } from "./config.js";

export type SetupCommand = { kind: "setup" } | { kind: "update"; spaceRepo?: string };

export function parseSetupCommand(argv: readonly string[]): SetupCommand {
if (argv.length === 0) return { kind: "setup" };
const [command, maybeSpaceRepo, ...extra] = argv;
if (command !== "update") {
throw new Error(`Unknown command: ${command ?? ""}. Use no arguments or "update".`);
}
if (extra.length > 0) throw new Error("Usage: npm run update -- [owner/xtap-pool]");
if (maybeSpaceRepo === undefined) return { kind: "update" };
const error = validateRepoId(maybeSpaceRepo);
if (error !== undefined) throw new Error(error);
return { kind: "update", spaceRepo: maybeSpaceRepo };
}
31 changes: 31 additions & 0 deletions setup/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,29 @@ export function defaultSetupConfig(username: string): SetupConfig {
};
}

export function existingSpaceConfig(
username: string,
spaceRepo: string,
variables: ReadonlyMap<string, string>,
): SetupConfig {
const spaceError = validateRepoId(spaceRepo);
if (spaceError !== undefined) throw new Error(spaceError);
const namespace = spaceRepo.split("/")[0] ?? "";
const datasetRepo = variables.get("DATASET_REPO") ?? repoInNamespace(namespace, "xtap-pool-data");
const datasetError = validateRepoId(datasetRepo);
if (datasetError !== undefined)
throw new Error(`Invalid DATASET_REPO on ${spaceRepo}: ${datasetRepo}`);
const allowedUsers = usersFromVariable(variables.get("ALLOWED_USERS"), [username]);
const poolAdmins = usersFromVariable(variables.get("POOL_ADMINS"), allowedUsers.slice(0, 1));
return {
namespace,
spaceRepo,
datasetRepo,
allowedUsers,
poolAdmins,
};
}

export function normalizeUsers(input: string): readonly string[] {
return [
...new Set(
Expand All @@ -30,6 +53,14 @@ export function normalizeUsers(input: string): readonly string[] {
];
}

function usersFromVariable(
value: string | undefined,
fallback: readonly string[],
): readonly string[] {
const parsed = value === undefined ? [] : normalizeUsers(value);
return parsed.length > 0 ? parsed : fallback;
}

export function usersValue(users: readonly string[]): string {
return users.join(",");
}
Expand Down
21 changes: 19 additions & 2 deletions setup/src/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { captureCommand } from "./process.js";
import { collectUploadFiles, createSpaceStage } from "./stage.js";

type DeleteOperation = { operation: "delete"; path: string };
type ConfigureSpaceOptions = { initializeGeneratedSecrets?: boolean };

export async function deployPool(
root: string,
Expand All @@ -30,7 +31,23 @@ export async function deployPool(
await configureSpace(client, config);
}

export async function configureSpace(client: HubClient, config: SetupConfig): Promise<void> {
export async function updateExistingPool(
root: string,
client: HubClient,
config: SetupConfig,
): Promise<void> {
await assertRepoVisibility(client, { type: "dataset", name: config.datasetRepo }, "private");
await assertRepoVisibility(client, { type: "space", name: config.spaceRepo }, "public");
await uploadSpace(root, client, config.spaceRepo);
await configureSpace(client, config, { initializeGeneratedSecrets: false });
}

export async function configureSpace(
client: HubClient,
config: SetupConfig,
options: ConfigureSpaceOptions = {},
): Promise<void> {
const initializeGeneratedSecrets = options.initializeGeneratedSecrets ?? true;
const variables = await getSpaceVariables(client, config.spaceRepo);
await setSpaceVariable(client, config.spaceRepo, "DATASET_REPO", config.datasetRepo);
await setSpaceVariable(
Expand All @@ -40,7 +57,7 @@ export async function configureSpace(client: HubClient, config: SetupConfig): Pr
usersValue(config.allowedUsers),
);
await setSpaceVariable(client, config.spaceRepo, "POOL_ADMINS", usersValue(config.poolAdmins));
if (!variables.has("SECRETS_INITIALIZED")) {
if (initializeGeneratedSecrets && !variables.has("SECRETS_INITIALIZED")) {
await setSpaceSecret(client, config.spaceRepo, "POOL_SIGNING_SECRET", randomSecret());
await setSpaceSecret(client, config.spaceRepo, "SESSION_SECRET", randomSecret());
await setSpaceVariable(client, config.spaceRepo, "SECRETS_INITIALIZED", "1");
Expand Down
7 changes: 5 additions & 2 deletions setup/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ import { fileURLToPath } from "node:url";

import { cancel } from "@clack/prompts";

import { parseSetupCommand } from "./cli.js";
import { findProjectRoot } from "./root.js";
import { runSetupWizard } from "./wizard.js";
import { runSetupWizard, runUpdateCommand } from "./wizard.js";

const here = dirname(fileURLToPath(import.meta.url));
const root = findProjectRoot(here);

try {
await runSetupWizard(root);
const command = parseSetupCommand(process.argv.slice(2));
if (command.kind === "setup") await runSetupWizard(root);
else await runUpdateCommand(root, command.spaceRepo);
} catch (error) {
const message = error instanceof Error ? error.message : "unknown error";
cancel(message);
Expand Down
19 changes: 17 additions & 2 deletions setup/src/wizard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { whoAmI } from "@huggingface/hub";
import type { SetupConfig } from "./config.js";
import {
defaultSetupConfig,
existingSpaceConfig,
normalizeUsers,
repoInNamespace,
spacePublicUrl,
Expand All @@ -23,8 +24,8 @@ import {
validateRepoId,
validateUserList,
} from "./config.js";
import { deployPool } from "./deploy.js";
import { setSpaceSecret } from "./hub-api.js";
import { deployPool, updateExistingPool } from "./deploy.js";
import { getSpaceVariables, setSpaceSecret } from "./hub-api.js";
import { defaultTweetsDirectory, expandHomePath } from "./path.js";
import { captureCommand, inheritCommand } from "./process.js";
import { verifyDatasetWriteToken } from "./token.js";
Expand All @@ -45,6 +46,20 @@ export async function runSetupWizard(root: string): Promise<void> {
outro(`Done. Explorer: ${spacePublicUrl(config.spaceRepo)}`);
}

export async function runUpdateCommand(root: string, requestedSpaceRepo?: string): Promise<void> {
intro("xtap-pool update");
const accessToken = await activeHfToken();
const account = await whoAmI({ accessToken });
const spaceRepo = requestedSpaceRepo ?? repoInNamespace(account.name, "xtap-pool");
const variables = await getSpaceVariables({ accessToken }, spaceRepo);
const config = existingSpaceConfig(account.name, spaceRepo, variables);
const task = spinner();
task.start(`Updating ${config.spaceRepo}`);
await updateExistingPool(root, { accessToken }, config);
task.stop("Space updated");
outro(`Done. Explorer: ${spacePublicUrl(config.spaceRepo)}`);
}

async function activeHfToken(): Promise<string> {
const result = await captureCommand("hf", ["auth", "token", "--quiet"]);
const token = result.stdout.trim();
Expand Down
23 changes: 23 additions & 0 deletions setup/tests/cli.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { describe, expect, it } from "vitest";

import { parseSetupCommand } from "../src/cli.js";

describe("setup CLI command parsing", () => {
it("defaults to the interactive setup wizard", () => {
expect(parseSetupCommand([])).toEqual({ kind: "setup" });
});

it("parses update mode with an optional Space repo", () => {
expect(parseSetupCommand(["update"])).toEqual({ kind: "update" });
expect(parseSetupCommand(["update", "alice/xtap-pool"])).toEqual({
kind: "update",
spaceRepo: "alice/xtap-pool",
});
});

it("rejects unknown commands and invalid update arguments", () => {
expect(() => parseSetupCommand(["deploy"])).toThrow("Unknown command");
expect(() => parseSetupCommand(["update", "not-a-repo"])).toThrow("owner/name");
expect(() => parseSetupCommand(["update", "alice/xtap-pool", "extra"])).toThrow("Usage");
});
});
33 changes: 33 additions & 0 deletions setup/tests/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";

import {
defaultSetupConfig,
existingSpaceConfig,
normalizeUsers,
repoInNamespace,
spacePublicUrl,
Expand All @@ -28,6 +29,38 @@ describe("setup config helpers", () => {
expect(usersValue(["alice", "bob"])).toBe("alice,bob");
});

it("derives update config from existing Space variables", () => {
const config = existingSpaceConfig(
"alice",
"team/xtap-pool",
new Map([
["DATASET_REPO", "team/tweets"],
["ALLOWED_USERS", "alice,bob"],
["POOL_ADMINS", "bob"],
]),
);

expect(config).toEqual({
namespace: "team",
spaceRepo: "team/xtap-pool",
datasetRepo: "team/tweets",
allowedUsers: ["alice", "bob"],
poolAdmins: ["bob"],
});
});

it("uses sane update defaults when optional Space variables are missing", () => {
const config = existingSpaceConfig("alice", "team/xtap-pool", new Map());

expect(config).toEqual({
namespace: "team",
spaceRepo: "team/xtap-pool",
datasetRepo: "team/xtap-pool-data",
allowedUsers: ["alice"],
poolAdmins: ["alice"],
});
});

it("validates repo ids and user lists", () => {
expect(validateNamespace("dutifuldev")).toBeUndefined();
expect(validateNamespace("bad namespace")).toContain("username or organization");
Expand Down
42 changes: 41 additions & 1 deletion setup/tests/deploy.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";

import { collectStaleSpaceDeletes } from "../src/deploy.js";
import { collectStaleSpaceDeletes, configureSpace } from "../src/deploy.js";

describe("setup deployment helpers", () => {
it("builds delete operations for remote Space files missing from the staged upload", async () => {
Expand Down Expand Up @@ -29,10 +29,50 @@ describe("setup deployment helpers", () => {
);
expect(new Headers(requests[0]?.init.headers).get("authorization")).toBe("Bearer hf_owner");
});

it("can update variables without initializing generated secrets", async () => {
const requests: { url: string; init: RequestInit }[] = [];
const fetchFn: typeof fetch = (input, init) => {
requests.push({ url: requestUrl(input), init: init ?? {} });
if (init?.method === "GET") return Promise.resolve(Response.json({}));
return Promise.resolve(new Response(null, { status: 204 }));
};

await configureSpace(
{ accessToken: "hf_owner", hubUrl: "https://hub.test", fetchFn },
{
namespace: "alice",
spaceRepo: "alice/xtap-pool",
datasetRepo: "alice/xtap-pool-data",
allowedUsers: ["alice"],
poolAdmins: ["alice"],
},
{ initializeGeneratedSecrets: false },
);

expect(requests.map((request) => [request.url, request.init.method])).toEqual([
["https://hub.test/api/spaces/alice/xtap-pool/variables", "GET"],
["https://hub.test/api/spaces/alice/xtap-pool/variables", "POST"],
["https://hub.test/api/spaces/alice/xtap-pool/variables", "POST"],
["https://hub.test/api/spaces/alice/xtap-pool/variables", "POST"],
]);
expect(requests.map((request) => requestBody(request.init))).toEqual([
undefined,
JSON.stringify({ key: "DATASET_REPO", value: "alice/xtap-pool-data" }),
JSON.stringify({ key: "ALLOWED_USERS", value: "alice" }),
JSON.stringify({ key: "POOL_ADMINS", value: "alice" }),
]);
});
});

function requestUrl(input: string | URL | Request): string {
if (typeof input === "string") return input;
if (input instanceof URL) return input.toString();
return input.url;
}

function requestBody(init: RequestInit): string | undefined {
if (init.body === undefined || init.body === null) return undefined;
if (typeof init.body === "string") return init.body;
throw new Error("expected string request body");
}
Loading