From f096c6c263ace3b4159d2c25a9520086b1b07527 Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Tue, 24 Jun 2025 13:30:50 +0200 Subject: [PATCH 1/7] Update promptfoo package from 0.79.0 to 0.115.1 This was done by running `nix-shell -p nix-update` and then from inside that shell, running `nix-update promptfoo`. --- pkgs/by-name/pr/promptfoo/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index c8dd41207455e..6f4648528c8d4 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -6,16 +6,16 @@ buildNpmPackage rec { pname = "promptfoo"; - version = "0.79.0"; + version = "0.115.1"; src = fetchFromGitHub { owner = "promptfoo"; repo = "promptfoo"; rev = "${version}"; - hash = "sha256-sMBgjxPzG3SJ7RS4oTtOq7hJ1MYaKW3/6FF8Pn5l89c="; + hash = "sha256-KGi5IV4R0WOvshTrVaVN3u0pO8as/A4/858JlFOdc0c="; }; - npmDepsHash = "sha256-tnzeEFEc/BMN/VsoNHWJIWDOvupHfddqI6020Q4M0RM="; + npmDepsHash = "sha256-vkZx2c5vvjHM8kSLukL7YQwRLnwSdFLa+xKhn+ekuts="; dontNpmBuild = true; From f0494d011099bec644f0c5c06aceca5d27a8f999 Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Tue, 24 Jun 2025 13:39:20 +0200 Subject: [PATCH 2/7] Prevent Playwright being downloaded during build of the package This solves the following error: ``` ... error: builder for '/nix/store/10zk098dm6s6wxj73a3vfd6bqklywbq9-promptfoo-0.115.1.drv' failed with exit code 1; last 25 log lines: > npm error } > npm error Error: unable to get local issuer certificate > npm error at TLSSocket.onConnectSecure (node:_tls_wrap:1679:34) > npm error at TLSSocket.emit (node:events:518:28) > npm error at TLSSocket._finishInit (node:_tls_wrap:1078:8) > npm error at ssl.onhandshakedone (node:_tls_wrap:864:12) { > npm error code: 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY' > npm error } > npm error /private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/index.js:1096 > npm error throw new Error(`Failed to download ${title}, caused by > npm error ^ > npm error > npm error Error: Failed to download Chromium 136.0.7103.25 (playwright build v1169), caused by > npm error Error: Download failure, code=1 > npm error at ChildProcess. (/private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/browserFetcher.js:94:32) > npm error at ChildProcess.emit (node:events:518:28) > npm error at ChildProcess._handle.onexit (node:internal/child_process:293:12) > npm error at /private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/index.js:1096:13 > npm error at async Registry._downloadExecutable (/private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/index.js:1095:5) > npm error at async Registry.install (/private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/index.js:989:9) > npm error at async installBrowsersForNpmInstall (/private/tmp/nix-build-promptfoo-0.115.1.drv-0/source/node_modules/playwright-core/lib/server/registry/index.js:1220:3) > npm error > npm error Node.js v22.14.0 > npm error Log files were not written due to an error writing to the directory: /nix/store/k6gxiw6p3rfy2xwv1r2z1z5za4sr8r15-promptfoo-0.115.1-npm-deps/_logs > npm error You can rerun the command with `--loglevel=verbose` to see the logs in your terminal For full logs, run 'nix log /nix/store/10zk098dm6s6wxj73a3vfd6bqklywbq9-promptfoo-0.115.1.drv'. ``` --- pkgs/by-name/pr/promptfoo/package.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index 6f4648528c8d4..1e2b832258198 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -27,4 +27,7 @@ buildNpmPackage rec { license = lib.licenses.mit; maintainers = [ lib.maintainers.nathanielbrough ]; }; + + # Prevent Playwright download as that will cause the package build to fail + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; } From a3bc0c515d3f0668a8c0c0d621ea9cb2f12c2e46 Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Tue, 24 Jun 2025 13:43:36 +0200 Subject: [PATCH 3/7] Remove broken symlinks after install and prior to packaging This solves the following error: ``` ... error: builder for '/nix/store/jjn5jckyrbbf1lbl7sy3jylhkvckg6vl-promptfoo-0.115.1.drv' failed with exit code 1; last 25 log lines: > patching script interpreter paths in node_modules > node_modules/better-sqlite3/build/gyp-mac-tool: interpreter directive changed from "#!/usr/bin/env python3" to "/nix/store/vfdk6q81hdjqjfiqz8f92hibdck3kmn6-python3-3.12.10/bin/python3" > Finished npmConfigHook > patchPhase completed in 38 seconds > Running phase: updateAutotoolsGnuConfigScriptsPhase > Running phase: configurePhase > no configure script, doing nothing > Running phase: buildPhase > no Makefile or custom buildPhase, doing nothing > Running phase: installPhase > Executing npmInstallHook > > up to date, audited 748 packages in 9s > > 87 packages are looking for funding > run `npm fund` for details > > found 0 vulnerabilities > Finished npmInstallHook > Running phase: fixupPhase > checking for references to /private/tmp/nix-build-promptfoo-0.115.1.drv-0/ in /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1... > patching script interpreter paths in /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1 > ERROR: noBrokenSymlinks: the symlink /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1/lib/node_modules/promptfoo/node_modules/app points to a missing target: /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1/lib/node_modules/promptfoo/src/app > ERROR: noBrokenSymlinks: the symlink /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1/lib/node_modules/promptfoo/node_modules/promptfoo-docs points to a missing target: /nix/store/p8ps9pmbm3mkgjh1z5y250wpdcwfwcrw-promptfoo-0.115.1/lib/node_modules/promptfoo/site > ERROR: noBrokenSymlinks: found 2 dangling symlinks, 0 reflexive symlinks and 0 unreadable symlinks For full logs, run 'nix log /nix/store/jjn5jckyrbbf1lbl7sy3jylhkvckg6vl-promptfoo-0.115.1.drv'. ``` --- pkgs/by-name/pr/promptfoo/package.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index 1e2b832258198..f386ef8c5edde 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -30,4 +30,10 @@ buildNpmPackage rec { # Prevent Playwright download as that will cause the package build to fail PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; + + # Remove broken symlinks after installation + postInstall = '' + rm -f $out/lib/node_modules/promptfoo/node_modules/app + rm -f $out/lib/node_modules/promptfoo/node_modules/promptfoo-docs + ''; } From b68615dc6346c508a7afe9fd50951f1c5050b4f9 Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Wed, 25 Jun 2025 14:15:24 +0200 Subject: [PATCH 4/7] Removing `dontNpmBuild` to ensure promptfoo is installed properly Without this, promptfoo would fail to run with this error: ``` % promptfoo --version node:internal/modules/cjs/loader:1404 throw err; ^ Error: Cannot find module '/nix/store/hxl79bpga73qz4z6pcxd903b6b0xbnrb-promptfoo-0.115.3/lib/node_modules/promptfoo/dist/src/main.js' at Function._resolveFilename (node:internal/modules/cjs/loader:1401:15) at defaultResolveImpl (node:internal/modules/cjs/loader:1057:19) at resolveForCJSWithHooks (node:internal/modules/cjs/loader:1062:22) at Function._load (node:internal/modules/cjs/loader:1211:37) at TracingChannel.traceSync (node:diagnostics_channel:322:14) at wrapModuleLoad (node:internal/modules/cjs/loader:235:24) at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5) at node:internal/main/run_main_module:36:49 { code: 'MODULE_NOT_FOUND', requireStack: [] } ``` --- pkgs/by-name/pr/promptfoo/package.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index f386ef8c5edde..27c059664b915 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -17,8 +17,6 @@ buildNpmPackage rec { npmDepsHash = "sha256-vkZx2c5vvjHM8kSLukL7YQwRLnwSdFLa+xKhn+ekuts="; - dontNpmBuild = true; - meta = { description = "Test your prompts, models, RAGs. Evaluate and compare LLM outputs, catch regressions, and improve prompt quality"; mainProgram = "promptfoo"; From 22fbe0fa74c0287ead756f20950782e26948d511 Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Tue, 24 Jun 2025 16:41:29 +0200 Subject: [PATCH 5/7] Use `env = { ... }` for setting `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = 1` Also, remove redundant comment. --- pkgs/by-name/pr/promptfoo/package.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index 27c059664b915..c32bd558bbb3d 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -26,8 +26,9 @@ buildNpmPackage rec { maintainers = [ lib.maintainers.nathanielbrough ]; }; - # Prevent Playwright download as that will cause the package build to fail - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; + env = { + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = 1; + }; # Remove broken symlinks after installation postInstall = '' From efd31db94d92255be5d95e95491053c4f965e21c Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Tue, 24 Jun 2025 16:48:38 +0200 Subject: [PATCH 6/7] Remove `-f` from `rm` in `postInstall` Also, remove redundant comment. --- pkgs/by-name/pr/promptfoo/package.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index c32bd558bbb3d..894080d0ad375 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -30,9 +30,8 @@ buildNpmPackage rec { PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = 1; }; - # Remove broken symlinks after installation postInstall = '' - rm -f $out/lib/node_modules/promptfoo/node_modules/app - rm -f $out/lib/node_modules/promptfoo/node_modules/promptfoo-docs + rm $out/lib/node_modules/promptfoo/node_modules/app + rm $out/lib/node_modules/promptfoo/node_modules/promptfoo-docs ''; } From 3ca9d64e7cdbda5155646e9ae704040e86afa62c Mon Sep 17 00:00:00 2001 From: Roeland van Houte Date: Wed, 25 Jun 2025 14:17:08 +0200 Subject: [PATCH 7/7] Update promptfoo to version 0.115.3 This was done by running `nix-shell -p nix-update` and then from inside that shell, running `nix-update promptfoo`. --- pkgs/by-name/pr/promptfoo/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/pr/promptfoo/package.nix b/pkgs/by-name/pr/promptfoo/package.nix index 894080d0ad375..aafa927a8937a 100644 --- a/pkgs/by-name/pr/promptfoo/package.nix +++ b/pkgs/by-name/pr/promptfoo/package.nix @@ -6,16 +6,16 @@ buildNpmPackage rec { pname = "promptfoo"; - version = "0.115.1"; + version = "0.115.3"; src = fetchFromGitHub { owner = "promptfoo"; repo = "promptfoo"; rev = "${version}"; - hash = "sha256-KGi5IV4R0WOvshTrVaVN3u0pO8as/A4/858JlFOdc0c="; + hash = "sha256-JBDz3kb++oR84mcU5Fxk28NCc5OfLThvBpAD7GiMZx4="; }; - npmDepsHash = "sha256-vkZx2c5vvjHM8kSLukL7YQwRLnwSdFLa+xKhn+ekuts="; + npmDepsHash = "sha256-1SfaX3XrmSVXpZKuSbM4LdO7F4d0NCpa1QcpasEMoBw="; meta = { description = "Test your prompts, models, RAGs. Evaluate and compare LLM outputs, catch regressions, and improve prompt quality";