diff --git a/.changeset/cache-options-for-build.md b/.changeset/cache-options-for-build.md new file mode 100644 index 00000000..1c99a324 --- /dev/null +++ b/.changeset/cache-options-for-build.md @@ -0,0 +1,10 @@ +--- +"@cartesi/cli": minor +--- + +add --cache-from and --cache-to options to cartesi build + +Expose Docker Buildx cache backend options via `--cache-from ` and +`--cache-to ` CLI flags (repeatable). The same options can also be +set per-drive in `cartesi.toml` via `cache_from` and `cache_to` arrays. +CLI flags override the TOML values when provided. diff --git a/.github/workflows/cli.yaml b/.github/workflows/cli.yaml index 2f1701b2..4220c445 100644 --- a/.github/workflows/cli.yaml +++ b/.github/workflows/cli.yaml @@ -35,6 +35,12 @@ jobs: - name: Build run: bun run build --filter @cartesi/cli + - name: Set up QEMU + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: Test run: bun test apps/cli/ diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index 3e2192e8..1ddb95d7 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -6,7 +6,13 @@ import { genext2fs, mksquashfs } from "../exec/index.js"; type ImageBuildOptions = Pick< DockerDriveConfig, - "buildArgs" | "context" | "dockerfile" | "tags" | "target" + | "buildArgs" + | "cacheFrom" + | "cacheTo" + | "context" + | "dockerfile" + | "tags" + | "target" > & { destination: string; dockerfileContent?: string }; type ImageInfo = { @@ -22,6 +28,8 @@ type ImageInfo = { const buildImage = async (options: ImageBuildOptions): Promise => { const { buildArgs, + cacheFrom, + cacheTo, context, destination, dockerfile, @@ -52,6 +60,10 @@ const buildImage = async (options: ImageBuildOptions): Promise => { // set build args args.push(...buildArgs.flatMap((arg) => ["--build-arg", arg])); + // set cache options + args.push(...cacheFrom.flatMap((spec) => ["--cache-from", spec])); + args.push(...cacheTo.flatMap((spec) => ["--cache-to", spec])); + if (target) { args.push("--target", target); } diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index 958fa08d..83d63be9 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -17,6 +17,8 @@ import { bootMachine } from "../machine.js"; // context for Listr build tasks interface BuildContext { + cacheFrom: string[]; + cacheTo: string[]; config: Config; debug: boolean; destination: string; @@ -37,6 +39,8 @@ const buildDriveTask = ( break; } case "docker": { + if (ctx.cacheFrom.length > 0) drive.cacheFrom = ctx.cacheFrom; + if (ctx.cacheTo.length > 0) drive.cacheTo = ctx.cacheTo; const imageInfo = await buildDocker( name, drive, @@ -86,10 +90,22 @@ export const createBuildCommand = () => { .default(false) .hideHelp(), ) + .option( + "--cache-from ", + "cache source for docker buildx build (can be repeated)", + (value, prev) => prev.concat([value]), + [], + ) + .option( + "--cache-to ", + "cache destination for docker buildx build (can be repeated)", + (value, prev) => prev.concat([value]), + [], + ) .option("-d, --drives-only", "only build drives, do not boot machine") .option("-v, --verbose", "verbose output", false) .action(async (options) => { - const { debug, drivesOnly, verbose } = options; + const { cacheFrom, cacheTo, debug, drivesOnly, verbose } = options; // clean up temp files we create along the process tmp.setGracefulCleanup(); @@ -104,7 +120,14 @@ export const createBuildCommand = () => { await fs.emptyDir(destination); // XXX: make it less error prone // build context - const ctx = { config, debug, destination, imageInfo: undefined }; + const ctx = { + cacheFrom, + cacheTo, + config, + debug, + destination, + imageInfo: undefined, + }; // tasks to build drives const driveTasks = Object.entries(config.drives).map( diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index bf0397a4..669827b3 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -99,6 +99,8 @@ export type DirectoryDriveConfig = { export type DockerDriveConfig = { builder: "docker"; buildArgs: string[]; // default is empty array + cacheFrom: string[]; // default is empty array + cacheTo: string[]; // default is empty array context: string; dockerfile: string; extraSize: number; // default is 0 (no extra size) @@ -163,6 +165,8 @@ type TomlTable = { [key: string]: TomlPrimitive }; export const defaultRootDriveConfig = (): DriveConfig => ({ builder: "docker", buildArgs: [], + cacheFrom: [], + cacheTo: [], context: ".", dockerfile: "Dockerfile", // file on current working directory extraSize: 0, @@ -408,6 +412,8 @@ const parseDrive = (drive: TomlPrimitive): DriveConfig => { case "docker": { const { build_args, + cache_from, + cache_to, context, dockerfile, extra_size, @@ -422,6 +428,8 @@ const parseDrive = (drive: TomlPrimitive): DriveConfig => { return { builder: "docker", buildArgs: parseStringArray(build_args), + cacheFrom: parseStringArray(cache_from), + cacheTo: parseStringArray(cache_to), image: parseOptionalString(image), context: parseString(context, "."), dockerfile: parseString(dockerfile, "Dockerfile"), diff --git a/apps/cli/tests/integration/builder/docker.test.ts b/apps/cli/tests/integration/builder/docker.test.ts index 0e4c3cbc..12c8bce5 100644 --- a/apps/cli/tests/integration/builder/docker.test.ts +++ b/apps/cli/tests/integration/builder/docker.test.ts @@ -6,8 +6,10 @@ import { expect, it, } from "bun:test"; +import crypto from "node:crypto"; import fs from "fs-extra"; import path from "node:path"; +import { execa } from "execa"; import { build } from "../../../src/builder/docker.js"; import type { DockerDriveConfig } from "../../../src/config.js"; import { setupIntegrationTests, TEST_SDK } from "../config.js"; @@ -36,6 +38,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: ".", dockerfile: "Dockerfile", extraSize: 0, @@ -53,6 +57,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "data"), dockerfile: "Dockerfile", extraSize: 0, @@ -70,6 +76,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -88,6 +96,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -106,6 +116,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -120,3 +132,126 @@ describe("when building with the docker builder", () => { expect(stat.size).toEqual(29327360); }); }); + +describe("when using cache options", () => { + const image = TEST_SDK; + const CACHE_TEST_TAG = "cartesi-cli-integration-test-cache:latest"; + let destination: string; + + beforeEach(async () => { + destination = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(destination); + }); + + it( + "should populate local cache when --cache-to is specified", + async () => { + const cacheDir = path.join(destination, "cache"); + const drive: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [], + cacheTo: [`type=local,dest=${cacheDir}`], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [CACHE_TEST_TAG], + target: undefined, + }; + + await build("root", drive, image, destination, false); + + // Clean up the tagged image + await execa("docker", ["image", "rm", CACHE_TEST_TAG], { + reject: false, + }); + + // Cache directory must have been written + expect(fs.existsSync(path.join(cacheDir, "index.json"))).toBe(true); + }, + { timeout: 120000 }, + ); + + it( + "should recover layers from local cache when --cache-from is specified", + async () => { + const cacheDir = path.join(destination, "cache"); + + // ── Build 1: populate local cache ──────────────────────────────── + const driveWithCacheTo: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [], + cacheTo: [`type=local,dest=${cacheDir}`], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [CACHE_TEST_TAG], + target: undefined, + }; + await build("root", driveWithCacheTo, image, destination, false); + + const hash1 = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(destination, "root.ext2"))) + .digest("hex"); + + // ── Teardown: remove image from daemon and build cache ──────────── + // Remove tagged image so BuildKit cannot reuse its layers implicitly + await execa("docker", ["image", "rm", CACHE_TEST_TAG], { + reject: false, + }); + // Remove all build-cache records from the daemon + await execa("docker", ["builder", "prune", "--force"]); + + // Remove build artefacts so the second build starts clean + await fs.remove(path.join(destination, "root.ext2")); + + // ── Build 2: rebuild using only the local cache ────────────────── + const driveWithCacheFrom: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [`type=local,src=${cacheDir}`], + cacheTo: [], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [], + target: undefined, + }; + await build("root", driveWithCacheFrom, image, destination, false); + + const hash2 = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(destination, "root.ext2"))) + .digest("hex"); + + // ── Attest: cache was recovered ────────────────────────────────── + // Identical hashes mean the RUN date layer was reused (not re-executed). + // A fresh build would produce a different timestamp → different content → different hash. + expect(hash2).toEqual(hash1); + }, + { timeout: 180000 }, + ); +}); diff --git a/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache b/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache new file mode 100644 index 00000000..09b14628 --- /dev/null +++ b/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache @@ -0,0 +1,3 @@ +FROM --platform=linux/riscv64 ubuntu:24.04@sha256:3f83fb03282ef4e453bdf0060e0d83833bb3cf6e6f36f54d9b8517d311d78e03 +# Non-deterministic RUN: output changes each build unless the layer is reused from cache +RUN date +%s%N > /build-id.txt diff --git a/apps/cli/tests/unit/config.test.ts b/apps/cli/tests/unit/config.test.ts index d57ace91..ae224c10 100644 --- a/apps/cli/tests/unit/config.test.ts +++ b/apps/cli/tests/unit/config.test.ts @@ -78,6 +78,8 @@ shared = true`, root: { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], dockerfile: "backend/Dockerfile", context: ".", extraSize: 0, @@ -93,6 +95,19 @@ shared = true`, }); }); + it("should parse cache_from and cache_to for docker drive", () => { + const config = parse([ + `[drives.root] +builder = "docker" +cache_from = ["type=gha"] +cache_to = ["type=gha,mode=max"]`, + ]); + expect(config.drives.root).toMatchObject({ + cacheFrom: ["type=gha"], + cacheTo: ["type=gha,mode=max"], + }); + }); + /** * [machine] */ diff --git a/packages/devnet/build.ts b/packages/devnet/build.ts index 0b5da822..8e64e775 100644 --- a/packages/devnet/build.ts +++ b/packages/devnet/build.ts @@ -56,7 +56,7 @@ const dependencies: ListrTask[] = [ task: async () => await downloadAndExtract(file), })); -type ContractDeployments = Record; +type ContractDeployments = Record; /** * Collect contracts from deployments, objects keyed by contractName, with abi and address @@ -100,7 +100,7 @@ const collectContracts = async (dir: string): Promise => { contracts[contractName] = { abi, address }; return contracts; }, - {} as Record, + {} as Record, ); };