From d9e0aa66ed5cc01fad4662f44598071cc17fa8dd Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Thu, 30 Oct 2025 17:51:34 -0600 Subject: [PATCH 01/41] init with very basic flake.nix Signed-off-by: Chuck Adams --- .gitignore | 6 ++++++ README.md | 12 +++++++++++ flake.lock | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 25 ++++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 .gitignore create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36c59e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.* + +build/ +node_modules/ +tmp/ +vendor/ diff --git a/README.md b/README.md index 4d9c4f2..ceffe52 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ # aspirebuild + +## Requirements + +* Nix, with flakes support. I _highly_ recommend the [Determinate Systems installer](https://docs.determinate.systems/) +* Recommended: `direnv` (should be available on apt/brew/dnf/pacman/etc) + +## Quick Start + +``` +echo "use flake" > .envrc && direnv allow # if using direnv +nix develop +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..6699936 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1761864852, + "narHash": "sha256-Wmpt+cRwQ8O5r9gGmkls2xOSBwqs1Og5qMWrREHTRv4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a5e47a4bea3996a6511f1da3cf6ba92e71a95f04", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a5e47a4bea3996a6511f1da3cf6ba92e71a95f04", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..19e84b5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,25 @@ +{ + inputs = { +# nixpkgs.url = "github:nixos/nixpkgs"; +# nixpkgs.url = "github:nixos/nixpkgs/25.05"; +# nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + nixpkgs.url = "github:nixos/nixpkgs/a5e47a4bea3996a6511f1da3cf6ba92e71a95f04"; # 2025-10-30 + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in { + packages.php = pkgs.php; + devShell = pkgs.mkShell { + buildInputs = [ + pkgs.curl + pkgs.php + pkgs.php84Packages.composer + ]; + }; + } + ); +} From 6847c2aeac5bd781e97b600613fa916714fb24e8 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Thu, 30 Oct 2025 18:26:05 -0600 Subject: [PATCH 02/41] add more packages to the base flake Signed-off-by: Chuck Adams --- flake.nix | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 19e84b5..b4a58e3 100644 --- a/flake.nix +++ b/flake.nix @@ -12,12 +12,22 @@ let pkgs = nixpkgs.legacyPackages.${system}; in { - packages.php = pkgs.php; - devShell = pkgs.mkShell { + devShell = with pkgs; mkShell { buildInputs = [ - pkgs.curl - pkgs.php - pkgs.php84Packages.composer + bash + coreutils + curl + gnutar + jq + just + lrzip + php + php84Packages.composer + subversion + sqlite + tzdata + zip + zstd ]; }; } From 3b77a8d27cfe8fa00811dcc2d0d7b8249bd65290 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Thu, 30 Oct 2025 19:18:46 -0600 Subject: [PATCH 03/41] add .editorconfig to git Signed-off-by: Chuck Adams --- .editorconfig | 15 +++++++++++++++ .gitignore | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c2f271c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false + +[*.{nix,yml,yaml}] +indent_size = 2 diff --git a/.gitignore b/.gitignore index 36c59e8..8ebc30a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# ignore all dotfiles and dotdirs by default, with exceptions below .* +!.gitignore +!.editorconfig build/ node_modules/ From 50d60d23582fdb3bff54f86dacced2006cbaf748 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 09:54:16 -0600 Subject: [PATCH 04/41] add perl to /flake.nix Signed-off-by: Chuck Adams --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index b4a58e3..34071b0 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,7 @@ jq just lrzip + perl php php84Packages.composer subversion From a452b4505fc8fdb830964582e377ea3e3a061aea Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 10:00:10 -0600 Subject: [PATCH 05/41] pin flake-utils to hash Signed-off-by: Chuck Adams --- flake.lock | 1 + flake.nix | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 6699936..e79ec35 100644 --- a/flake.lock +++ b/flake.lock @@ -15,6 +15,7 @@ "original": { "owner": "numtide", "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 34071b0..99077b2 100644 --- a/flake.nix +++ b/flake.nix @@ -3,8 +3,8 @@ # nixpkgs.url = "github:nixos/nixpkgs"; # nixpkgs.url = "github:nixos/nixpkgs/25.05"; # nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; - nixpkgs.url = "github:nixos/nixpkgs/a5e47a4bea3996a6511f1da3cf6ba92e71a95f04"; # 2025-10-30 - flake-utils.url = "github:numtide/flake-utils"; + nixpkgs.url = "github:nixos/nixpkgs/a5e47a4bea3996a6511f1da3cf6ba92e71a95f04"; # (2025-10-30) + flake-utils.url = "github:numtide/flake-utils/11707dc2f618dd54ca8739b309ec4fc024de578b"; # (2024-11-13) }; outputs = { self, nixpkgs, flake-utils }: From 61d3be4f7f40a308abf3211d0caa16d28acb250a Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 10:00:34 -0600 Subject: [PATCH 06/41] add legacy AS scripts Signed-off-by: Chuck Adams --- .../legacy-aspiresync/bin/ls-updated-plugins | 13 +++ tools/legacy-aspiresync/bin/ls-updated-themes | 13 +++ tools/legacy-aspiresync/bin/prelude.bash | 62 ++++++++++ tools/legacy-aspiresync/bin/reset-plugins | 3 + tools/legacy-aspiresync/bin/reset-themes | 4 + tools/legacy-aspiresync/bin/svn-changed-files | 10 ++ tools/legacy-aspiresync/bin/svn-changed-slugs | 11 ++ .../legacy-aspiresync/bin/svn-get-immediates | 12 ++ .../legacy-aspiresync/bin/update-plugin-trunk | 100 ++++++++++++++++ .../bin/update-theme-snapshot | 108 ++++++++++++++++++ 10 files changed, 336 insertions(+) create mode 100755 tools/legacy-aspiresync/bin/ls-updated-plugins create mode 100755 tools/legacy-aspiresync/bin/ls-updated-themes create mode 100644 tools/legacy-aspiresync/bin/prelude.bash create mode 100755 tools/legacy-aspiresync/bin/reset-plugins create mode 100755 tools/legacy-aspiresync/bin/reset-themes create mode 100755 tools/legacy-aspiresync/bin/svn-changed-files create mode 100755 tools/legacy-aspiresync/bin/svn-changed-slugs create mode 100755 tools/legacy-aspiresync/bin/svn-get-immediates create mode 100755 tools/legacy-aspiresync/bin/update-plugin-trunk create mode 100755 tools/legacy-aspiresync/bin/update-theme-snapshot diff --git a/tools/legacy-aspiresync/bin/ls-updated-plugins b/tools/legacy-aspiresync/bin/ls-updated-plugins new file mode 100755 index 0000000..f0d396b --- /dev/null +++ b/tools/legacy-aspiresync/bin/ls-updated-plugins @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +. $(dirname $0)/prelude.bash + +date=$1 +[[ -n $date ]] || die "usage: $0 " + +checkout=$(mktemp -d $TMPDIR/plugins-svn.XXXXXXXX) +trap "rm -rf $checkout" EXIT + +RUN cd $checkout +RUN svn co https://plugins.svn.wordpress.org --depth empty > /dev/null +RUN $BASE_DIR/bin/svn-changed-slugs $date diff --git a/tools/legacy-aspiresync/bin/ls-updated-themes b/tools/legacy-aspiresync/bin/ls-updated-themes new file mode 100755 index 0000000..6cca519 --- /dev/null +++ b/tools/legacy-aspiresync/bin/ls-updated-themes @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +. $(dirname $0)/prelude.bash + +date=$1 +[[ -n $date ]] || die "usage: $0 " + +checkout=$(mktemp -d $TMPDIR/themes-svn.XXXXXXXX) +trap "rm -rf $checkout" EXIT + +RUN cd $checkout +RUN svn co https://themes.svn.wordpress.org --depth empty > /dev/null +RUN $BASE_DIR/bin/svn-changed-slugs $date diff --git a/tools/legacy-aspiresync/bin/prelude.bash b/tools/legacy-aspiresync/bin/prelude.bash new file mode 100644 index 0000000..aa1e7c6 --- /dev/null +++ b/tools/legacy-aspiresync/bin/prelude.bash @@ -0,0 +1,62 @@ +# This file should be sourced, not run + +[[ -n $TRACE ]] && [[ $TRACE != 0 ]] && set -x + +set -o errexit + +ORIG_PWD=$(pwd) +cd $(dirname $0)/.. +BASE_DIR=$(pwd) + +DATA_DIR=${DATA_DIR:-$HOME/svn-data} # should NOT be under the project root, it freaks IDEA out even if its excluded +ARCHIVE_DIR=${ARCHIVE_DIR:-$DATA_DIR/archive} + +TMPDIR=${TMPDIR:-/tmp} # no underscore on this one, it's an old unixism + +mkdir -p $DATA_DIR $ARCHIVE_DIR + +PLUGINS_REMOTE=${PLUGINS_REMOTE:-https://plugins.svn.wordpress.org} +THEMES_REMOTE=${THEMES_REMOTE:-https://themes.svn.wordpress.org} + +YMD=$(date +%Y-%m-%d) + +function warn { + echo "$@" >&2 +} + +function die() { + warn "$@" + exit 1 +} + +function RUN() { + [[ -n $DRY_RUN ]] && [[ $DRY_RUN != 0 ]] && _run=echo + $_run "$@" +} + +function enforce_svn_root() { + [[ -d .svn ]] || die "$(pwd) does not look like a svn checkout -- exiting" +} + +function _use_checkout() { + type=$1 + remote=$2 + + cd $DATA_DIR + mkdir -p svn/$type + cd svn/$type + if [[ -d .svn ]]; then + $BASE_DIR/bin/svn-get-immediates + else + depth=${IMMEDIATES_DEPTH:-immediates} + svn checkout --ignore-externals --depth=$depth $remote + fi +} + +function use_plugins() { + _use_checkout plugins $PLUGINS_REMOTE +} + +function use_themes() { + _use_checkout themes $THEMES_REMOTE +} \ No newline at end of file diff --git a/tools/legacy-aspiresync/bin/reset-plugins b/tools/legacy-aspiresync/bin/reset-plugins new file mode 100755 index 0000000..18e9b09 --- /dev/null +++ b/tools/legacy-aspiresync/bin/reset-plugins @@ -0,0 +1,3 @@ +#!/bin/sh +. $(dirname $0)/prelude.bash +use_plugins diff --git a/tools/legacy-aspiresync/bin/reset-themes b/tools/legacy-aspiresync/bin/reset-themes new file mode 100755 index 0000000..9a7d607 --- /dev/null +++ b/tools/legacy-aspiresync/bin/reset-themes @@ -0,0 +1,4 @@ +#!/bin/sh +. $(dirname $0)/prelude.bash +use_themes + diff --git a/tools/legacy-aspiresync/bin/svn-changed-files b/tools/legacy-aspiresync/bin/svn-changed-files new file mode 100755 index 0000000..86e8e34 --- /dev/null +++ b/tools/legacy-aspiresync/bin/svn-changed-files @@ -0,0 +1,10 @@ +#!/bin/bash + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD +enforce_svn_root + +date=$1 +[[ -n $date ]] || die "Usage: $0 yyyy-mm-dd" + +RUN svn log --revision "{$date}:head" --verbose | perl -nE 'next unless m!^\s+[A-Z] (/.*)$/!; say $1' diff --git a/tools/legacy-aspiresync/bin/svn-changed-slugs b/tools/legacy-aspiresync/bin/svn-changed-slugs new file mode 100755 index 0000000..9cc4e15 --- /dev/null +++ b/tools/legacy-aspiresync/bin/svn-changed-slugs @@ -0,0 +1,11 @@ +#!/bin/bash + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD +enforce_svn_root + +date=$1 +[[ -n $date ]] || die "Usage: $0 yyyy-mm-dd" + +RUN svn log --revision "{$date}:head" --verbose | perl -nE 'next unless m!^\s+[A-Z] /(.*?)/!; say $1 unless $seen{$1}++' + diff --git a/tools/legacy-aspiresync/bin/svn-get-immediates b/tools/legacy-aspiresync/bin/svn-get-immediates new file mode 100755 index 0000000..41fcfb5 --- /dev/null +++ b/tools/legacy-aspiresync/bin/svn-get-immediates @@ -0,0 +1,12 @@ +#!/bin/sh + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD + +enforce_svn_root + +# some scripts set this to 'empty' +depth=${IMMEDIATES_DEPTH:-immediates} + +svn cleanup +svn update --ignore-externals --set-depth=$depth . diff --git a/tools/legacy-aspiresync/bin/update-plugin-trunk b/tools/legacy-aspiresync/bin/update-plugin-trunk new file mode 100755 index 0000000..398f6da --- /dev/null +++ b/tools/legacy-aspiresync/bin/update-plugin-trunk @@ -0,0 +1,100 @@ +#!/bin/bash + +# Must be run from within the svn checkout. Will create or update the archive directory using the same name as the current + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD + +enforce_svn_root + +repo=$(basename $(pwd)) +mkdir -p $ARCHIVE_DIR/$repo/trunks/ + +# These have so many redundant files (usually from recursively nested tags) that they're effectively zip bombs +bombs='all-in-one-contact-buttons-wpshare247|biblesupersearch' + +# These produce errors when svn attempts to update them +corrupt='a2-optimized-wp|better-links|countdown-timer|facebook-album-sync|font-awesome-the-easy-way|up-wp-cart' + +# combined blacklist +BLACKLIST="^($bombs|$corrupt)\$" + +#### + +function main() { + for dir in $* + do + process $dir + done +} + +function process() { + local dir=$1 + + echo -e "\nPROCESSING: $dir [trunk]" + + if ! [[ $dir =~ ^[-A-Za-z0-9]+$ ]]; then + finalize $dir "malformed dir ignored" + return + fi + + if [[ $dir =~ $BLACKLIST ]]; then + finalize $dir "blacklisted dir skipped" + return + fi + + [[ -d $dir ]] || svn update --ignore-externals --set-depth=empty $dir + + # Note: using a new /trunks base directory, but still using .trunk suffix so archive files can co-mingle + local rev=$(get_revision $dir) + local tar=$ARCHIVE_DIR/$repo/trunks/$rev/$rev.$dir.trunk.tar + local ztar=$tar.zst + + if [[ -f $ztar ]]; then + finalize $dir "archive exists at revision $rev" + return + fi + + mkdir -p $(dirname $tar) + update $dir + archive $dir $ztar + ls -lh $ztar + finalize $dir "archived at revision $rev" +} + +function update() { + local dir=$1 + svn update --set-depth=immediates --force $dir/tags # the list of versions is lightweight and useful metadata + svn update --ignore-externals --set-depth=infinity --force $dir/trunk +} + +function archive() { + local dir=$1 + local ztar=$2 + + tar cf - ./$dir | zstd --progress > $ztar.tmp + mv $ztar.tmp $ztar + rm -f $tar +} + +function finalize() { + local dir=$1 + local message=$2 + + warn "$dir: $message" + [[ -n $UPDATE_KEEP_DIRS ]] || rm -rf "$dir" +} + +function get_revision() { + local dir=$1 + local rev=$(svn info -r HEAD --show-item last-changed-revision $dir) + if [[ -z $rev ]]; then + die "fatal could not retrieve revision for $dir" + fi + echo $rev +} + +################ + +main $* + diff --git a/tools/legacy-aspiresync/bin/update-theme-snapshot b/tools/legacy-aspiresync/bin/update-theme-snapshot new file mode 100755 index 0000000..ab6f827 --- /dev/null +++ b/tools/legacy-aspiresync/bin/update-theme-snapshot @@ -0,0 +1,108 @@ +#!/bin/bash + +# Must be run from within the svn checkout. Will create or update the archive directory using the same name as the current + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD + +enforce_svn_root + +repo=$(basename $(pwd)) +mkdir -p $ARCHIVE_DIR/$repo/snapshots/ + +# probably not needed for themes, keeping it around anyway +bombs='xxxxxxxxxx' +corrupt='yyyyyyyyyy' +BLACKLIST="^($bombs|$corrupt)\$" + +#### + +function main() { + for dir in $* + do + process $dir + done +} + +function process() { + local dir=$1 + + echo -e "\nPROCESSING: $dir [trunk]" + + if ! [[ $dir =~ ^[-A-Za-z0-9]+$ ]]; then + finalize $dir "malformed dir ignored" + return + fi + + if [[ $dir =~ $BLACKLIST ]]; then + finalize $dir "blacklisted dir skipped" + return + fi + + svn update --ignore-externals --set-depth=immediates $dir + + local rev=$(get_revision $dir) + local ver=$(get_latest_version $dir) + + if [[ -z $ver ]]; then + finalize $dir "could not find latest version" + return + fi + + # not encoding $ver into filename, it should be stable at $rev anyway + local tar=$ARCHIVE_DIR/$repo/snapshots/$rev/$rev.$dir.snapshot.tar + local ztar=$tar.zst + + if [[ -f $ztar ]]; then + finalize $dir "archive exists at revision $rev" + return + fi + + mkdir -p $(dirname $tar) + update $dir $ver + archive $dir $ztar + ls -lh $ztar + finalize $dir "archived at revision $rev" +} + +function update() { + local dir=$1 + local ver=$2 + svn update --ignore-externals --set-depth=infinity --force $dir/$ver +} + +function archive() { + local dir=$1 + local ztar=$2 + + tar cf - ./$dir | zstd --progress > $ztar.tmp + mv $ztar.tmp $ztar + rm -f $tar +} + +function finalize() { + local dir=$1 + local message=$2 + + warn "$dir: $message" + [[ -n $UPDATE_KEEP_DIRS ]] || rm -rf "$dir" +} + +function get_revision() { + local dir=$1 + local rev=$(svn info -r HEAD --show-item last-changed-revision $dir) + if [[ -z $rev ]]; then + die "fatal could not retrieve revision for $dir" + fi + echo $rev +} + +function get_latest_version() { + local dir=$1 + ls $dir | php -r '$vs = explode("\n", file_get_contents("php://stdin")); $vs = array_filter($vs, fn($s) => preg_match("/^[0-9]/", $s)); usort($vs, fn($a, $b) => version_compare($b, $a)); echo $vs[0] ?? "";' +} + +################ + +main $* + From 438fea87865fab935007243dfc8bc6d5a18cd976 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 12:44:36 -0600 Subject: [PATCH 07/41] update prelude.bash with nounset and pipefail Signed-off-by: Chuck Adams --- tools/legacy-aspiresync/bin/prelude.bash | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/legacy-aspiresync/bin/prelude.bash b/tools/legacy-aspiresync/bin/prelude.bash index aa1e7c6..e6d0b25 100644 --- a/tools/legacy-aspiresync/bin/prelude.bash +++ b/tools/legacy-aspiresync/bin/prelude.bash @@ -3,6 +3,8 @@ [[ -n $TRACE ]] && [[ $TRACE != 0 ]] && set -x set -o errexit +set -o nounset +set -o pipefail ORIG_PWD=$(pwd) cd $(dirname $0)/.. @@ -13,10 +15,13 @@ ARCHIVE_DIR=${ARCHIVE_DIR:-$DATA_DIR/archive} TMPDIR=${TMPDIR:-/tmp} # no underscore on this one, it's an old unixism -mkdir -p $DATA_DIR $ARCHIVE_DIR - PLUGINS_REMOTE=${PLUGINS_REMOTE:-https://plugins.svn.wordpress.org} +PLUGINS_DIR=${PLUGINS_DIR:-$DATA_DIR/svn/plugins} + THEMES_REMOTE=${THEMES_REMOTE:-https://themes.svn.wordpress.org} +THEMES_DIR=${THEMES_DIR:-$DATA_DIR/svn/themes} + +mkdir -p $DATA_DIR $ARCHIVE_DIR $PLUGINS_DIR $THEMES_DIR YMD=$(date +%Y-%m-%d) @@ -30,7 +35,8 @@ function die() { } function RUN() { - [[ -n $DRY_RUN ]] && [[ $DRY_RUN != 0 ]] && _run=echo + local _run= + [[ -n ${DRY_RUN:-} ]] && [[ $DRY_RUN != 0 ]] && _run=echo $_run "$@" } @@ -59,4 +65,4 @@ function use_plugins() { function use_themes() { _use_checkout themes $THEMES_REMOTE -} \ No newline at end of file +} From b44116fe0ffa1d3a69310d6302dada229127597f Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 12:45:07 -0600 Subject: [PATCH 08/41] update scripts cd to PLUGINS_DIR and THEMES_DIR Signed-off-by: Chuck Adams --- tools/legacy-aspiresync/bin/update-plugin-trunk | 5 ++--- tools/legacy-aspiresync/bin/update-theme-snapshot | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/legacy-aspiresync/bin/update-plugin-trunk b/tools/legacy-aspiresync/bin/update-plugin-trunk index 398f6da..c67ac63 100755 --- a/tools/legacy-aspiresync/bin/update-plugin-trunk +++ b/tools/legacy-aspiresync/bin/update-plugin-trunk @@ -1,9 +1,8 @@ #!/bin/bash -# Must be run from within the svn checkout. Will create or update the archive directory using the same name as the current . $(dirname $0)/prelude.bash -cd $ORIG_PWD +cd $PLUGINS_DIR enforce_svn_root @@ -82,7 +81,7 @@ function finalize() { local message=$2 warn "$dir: $message" - [[ -n $UPDATE_KEEP_DIRS ]] || rm -rf "$dir" + [[ -n ${UPDATE_KEEP_DIRS:-} ]] || rm -rf "$dir" } function get_revision() { diff --git a/tools/legacy-aspiresync/bin/update-theme-snapshot b/tools/legacy-aspiresync/bin/update-theme-snapshot index ab6f827..c224e34 100755 --- a/tools/legacy-aspiresync/bin/update-theme-snapshot +++ b/tools/legacy-aspiresync/bin/update-theme-snapshot @@ -3,7 +3,7 @@ # Must be run from within the svn checkout. Will create or update the archive directory using the same name as the current . $(dirname $0)/prelude.bash -cd $ORIG_PWD +cd $THEMES_DIR enforce_svn_root From 4371ead1ef8f70568ffc5f2dfcf02825d3816177 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 13:54:51 -0600 Subject: [PATCH 09/41] add auto svn cleanup to update-plugin-trunk Signed-off-by: Chuck Adams --- tools/legacy-aspiresync/bin/update-plugin-trunk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/legacy-aspiresync/bin/update-plugin-trunk b/tools/legacy-aspiresync/bin/update-plugin-trunk index c67ac63..dc938ea 100755 --- a/tools/legacy-aspiresync/bin/update-plugin-trunk +++ b/tools/legacy-aspiresync/bin/update-plugin-trunk @@ -6,6 +6,8 @@ cd $PLUGINS_DIR enforce_svn_root +[[ -n ${NO_SVN_CLEANUP:-} ]] || svn cleanup + repo=$(basename $(pwd)) mkdir -p $ARCHIVE_DIR/$repo/trunks/ From dc6e6eb7c0a5ac43e92662829680c9befa06b5f0 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 16:16:10 -0600 Subject: [PATCH 10/41] pare down legacy scripts to archive-plugin Signed-off-by: Chuck Adams --- tools/legacy-aspiresync/bin/archive-plugin | 94 +++++++++++++++ tools/legacy-aspiresync/bin/reset-plugins | 3 - tools/legacy-aspiresync/bin/reset-themes | 4 - tools/legacy-aspiresync/bin/svn-changed-files | 0 tools/legacy-aspiresync/bin/svn-changed-slugs | 0 .../legacy-aspiresync/bin/svn-get-immediates | 12 -- .../legacy-aspiresync/bin/update-plugin-trunk | 1 - .../bin/update-theme-snapshot | 108 ------------------ 8 files changed, 94 insertions(+), 128 deletions(-) create mode 100755 tools/legacy-aspiresync/bin/archive-plugin delete mode 100755 tools/legacy-aspiresync/bin/reset-plugins delete mode 100755 tools/legacy-aspiresync/bin/reset-themes mode change 100755 => 100644 tools/legacy-aspiresync/bin/svn-changed-files mode change 100755 => 100644 tools/legacy-aspiresync/bin/svn-changed-slugs delete mode 100755 tools/legacy-aspiresync/bin/svn-get-immediates delete mode 100755 tools/legacy-aspiresync/bin/update-theme-snapshot diff --git a/tools/legacy-aspiresync/bin/archive-plugin b/tools/legacy-aspiresync/bin/archive-plugin new file mode 100755 index 0000000..b470153 --- /dev/null +++ b/tools/legacy-aspiresync/bin/archive-plugin @@ -0,0 +1,94 @@ +#!/bin/bash + +# shellcheck disable=SC2046 +. $(dirname "$0")/prelude.bash + +OUTPUT_DIR=$ARCHIVE_DIR/plugins/revisions +mkdir -p "$OUTPUT_DIR" + +# These have so many redundant files (usually from recursively nested tags) that they're effectively zip bombs +bombs='all-in-one-contact-buttons-wpshare247|biblesupersearch' + +# These produce errors when svn attempts to update them +corrupt='a2-optimized-wp|better-links|countdown-timer|facebook-album-sync|font-awesome-the-easy-way|up-wp-cart' + +# combined deny-list +DENYLIST="^($bombs|$corrupt)\$" + +#### + +function main() { + local slug=$1 + local tag=$2 + local rev=${3:-HEAD} + + echo "" + echo "PROCESSING: $slug [tag=$tag rev=$rev]" + + if ! [[ $slug =~ ^[-A-Za-z0-9]+$ ]]; then + die "malformed slug skipped: $slug" + return + fi + + if [[ $slug =~ $DENYLIST ]]; then + die "slug skipped due to matching explicit deny pattern: $slug" + return + fi + + enter_tmpdir + + svn co --depth=empty "$PLUGINS_REMOTE/$slug" + [[ $rev = 'HEAD' ]] && rev=$(get_revision "$slug") + local tar=$OUTPUT_DIR/$rev/$rev.$slug.$tag.tar.zst + + if [[ -f $tar ]] && [[ -z "${FORCE:-}" ]]; then + echo "archive exists for $slug [tag=$tag rev=$rev file=$tar]" + exit 0 + fi + + cd "$slug" || die "could not chdir to $(pwd)/$slug" + + svn update --set-depth=immediates --revision "$rev" tags + + if [[ $tag = 'trunk' ]]; then + svn update --set-depth=infinity --revision "$rev" trunk + else + [[ -d tags/$tag ]] || die "Could not find tag: $slug/tags/$tag" + svn update --set-depth=infinity --revision "$rev" "tags/$tag" + fi + + svn update --set-depth=infinity --revision "$rev" assets + + rm -rf .svn + cd .. + + mkdir -p $(dirname "$tar") + tar cf - "./$slug" | zstd --progress > "$tar.tmp" + mv "$tar.tmp" "$tar" + + echo "ARCHIVED: $slug [tag=$tag rev=$rev file=$tar]" +} + +function enter_tmpdir() { + local tmpdir + tmpdir=$(mktemp -d "$TMPDIR/svn.XXXXXXXX") + # shellcheck disable=SC2064 + trap "rm -rf $tmpdir" EXIT + cd "$tmpdir" || die "could not set pwd to $tmpdir" +} + +function get_revision() { + local dir=$1 + local rev + rev=$(svn info -r HEAD --show-item last-changed-revision "$dir") + + if [[ -z $rev ]]; then + die "fatal could not retrieve revision for $dir" + fi + echo "$rev" +} + +################ + +main "$@" + diff --git a/tools/legacy-aspiresync/bin/reset-plugins b/tools/legacy-aspiresync/bin/reset-plugins deleted file mode 100755 index 18e9b09..0000000 --- a/tools/legacy-aspiresync/bin/reset-plugins +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -. $(dirname $0)/prelude.bash -use_plugins diff --git a/tools/legacy-aspiresync/bin/reset-themes b/tools/legacy-aspiresync/bin/reset-themes deleted file mode 100755 index 9a7d607..0000000 --- a/tools/legacy-aspiresync/bin/reset-themes +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -. $(dirname $0)/prelude.bash -use_themes - diff --git a/tools/legacy-aspiresync/bin/svn-changed-files b/tools/legacy-aspiresync/bin/svn-changed-files old mode 100755 new mode 100644 diff --git a/tools/legacy-aspiresync/bin/svn-changed-slugs b/tools/legacy-aspiresync/bin/svn-changed-slugs old mode 100755 new mode 100644 diff --git a/tools/legacy-aspiresync/bin/svn-get-immediates b/tools/legacy-aspiresync/bin/svn-get-immediates deleted file mode 100755 index 41fcfb5..0000000 --- a/tools/legacy-aspiresync/bin/svn-get-immediates +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -. $(dirname $0)/prelude.bash -cd $ORIG_PWD - -enforce_svn_root - -# some scripts set this to 'empty' -depth=${IMMEDIATES_DEPTH:-immediates} - -svn cleanup -svn update --ignore-externals --set-depth=$depth . diff --git a/tools/legacy-aspiresync/bin/update-plugin-trunk b/tools/legacy-aspiresync/bin/update-plugin-trunk index dc938ea..d569656 100755 --- a/tools/legacy-aspiresync/bin/update-plugin-trunk +++ b/tools/legacy-aspiresync/bin/update-plugin-trunk @@ -75,7 +75,6 @@ function archive() { tar cf - ./$dir | zstd --progress > $ztar.tmp mv $ztar.tmp $ztar - rm -f $tar } function finalize() { diff --git a/tools/legacy-aspiresync/bin/update-theme-snapshot b/tools/legacy-aspiresync/bin/update-theme-snapshot deleted file mode 100755 index c224e34..0000000 --- a/tools/legacy-aspiresync/bin/update-theme-snapshot +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Must be run from within the svn checkout. Will create or update the archive directory using the same name as the current - -. $(dirname $0)/prelude.bash -cd $THEMES_DIR - -enforce_svn_root - -repo=$(basename $(pwd)) -mkdir -p $ARCHIVE_DIR/$repo/snapshots/ - -# probably not needed for themes, keeping it around anyway -bombs='xxxxxxxxxx' -corrupt='yyyyyyyyyy' -BLACKLIST="^($bombs|$corrupt)\$" - -#### - -function main() { - for dir in $* - do - process $dir - done -} - -function process() { - local dir=$1 - - echo -e "\nPROCESSING: $dir [trunk]" - - if ! [[ $dir =~ ^[-A-Za-z0-9]+$ ]]; then - finalize $dir "malformed dir ignored" - return - fi - - if [[ $dir =~ $BLACKLIST ]]; then - finalize $dir "blacklisted dir skipped" - return - fi - - svn update --ignore-externals --set-depth=immediates $dir - - local rev=$(get_revision $dir) - local ver=$(get_latest_version $dir) - - if [[ -z $ver ]]; then - finalize $dir "could not find latest version" - return - fi - - # not encoding $ver into filename, it should be stable at $rev anyway - local tar=$ARCHIVE_DIR/$repo/snapshots/$rev/$rev.$dir.snapshot.tar - local ztar=$tar.zst - - if [[ -f $ztar ]]; then - finalize $dir "archive exists at revision $rev" - return - fi - - mkdir -p $(dirname $tar) - update $dir $ver - archive $dir $ztar - ls -lh $ztar - finalize $dir "archived at revision $rev" -} - -function update() { - local dir=$1 - local ver=$2 - svn update --ignore-externals --set-depth=infinity --force $dir/$ver -} - -function archive() { - local dir=$1 - local ztar=$2 - - tar cf - ./$dir | zstd --progress > $ztar.tmp - mv $ztar.tmp $ztar - rm -f $tar -} - -function finalize() { - local dir=$1 - local message=$2 - - warn "$dir: $message" - [[ -n $UPDATE_KEEP_DIRS ]] || rm -rf "$dir" -} - -function get_revision() { - local dir=$1 - local rev=$(svn info -r HEAD --show-item last-changed-revision $dir) - if [[ -z $rev ]]; then - die "fatal could not retrieve revision for $dir" - fi - echo $rev -} - -function get_latest_version() { - local dir=$1 - ls $dir | php -r '$vs = explode("\n", file_get_contents("php://stdin")); $vs = array_filter($vs, fn($s) => preg_match("/^[0-9]/", $s)); usort($vs, fn($a, $b) => version_compare($b, $a)); echo $vs[0] ?? "";' -} - -################ - -main $* - From 0c8fe495ae086baff1704ac1cafe69b08693370f Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 31 Oct 2025 17:39:41 -0600 Subject: [PATCH 11/41] chmod +x bin/svn-changed-* Signed-off-by: Chuck Adams --- tools/legacy-aspiresync/bin/svn-changed-files | 0 tools/legacy-aspiresync/bin/svn-changed-slugs | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/legacy-aspiresync/bin/svn-changed-files mode change 100644 => 100755 tools/legacy-aspiresync/bin/svn-changed-slugs diff --git a/tools/legacy-aspiresync/bin/svn-changed-files b/tools/legacy-aspiresync/bin/svn-changed-files old mode 100644 new mode 100755 diff --git a/tools/legacy-aspiresync/bin/svn-changed-slugs b/tools/legacy-aspiresync/bin/svn-changed-slugs old mode 100644 new mode 100755 From 4dda143dee825639f8c4d2ad0dcec5abaa789eff Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 14:29:37 -0700 Subject: [PATCH 12/41] add some WIP docker image generation Signed-off-by: Chuck Adams --- flake.nix | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 99077b2..e21f113 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,21 @@ zstd ]; }; + + # WIP docker support + # https://community.flake.parts/haskell-flake/docker + packages = { + dockerImage = pkgs.dockerTools.buildImage { + name = "aspirebuild"; + tag = builtins.substring 0 9 (self.rev or "dev"); # tag with git revision, or 'dev' if dirty + config = { + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "SYSTEM_CERTIFICATE_PATH=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + }; + }; + }; } - ); + ); } From 656eaa4cdb09d12b77af1dc03124c81f98145b2c Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 14:50:28 -0700 Subject: [PATCH 13/41] start of mkbuilder tool Signed-off-by: Chuck Adams --- .gitignore | 8 ++++++- tools/_common/lib/bash/prelude.bash | 33 +++++++++++++++++++++++++++++ tools/mkbuilder/bin/builder-info | 6 ++++++ tools/mkbuilder/bin/mkbuilder | 13 ++++++++++++ tools/mkbuilder/lib/bash | 1 + 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tools/_common/lib/bash/prelude.bash create mode 100755 tools/mkbuilder/bin/builder-info create mode 100755 tools/mkbuilder/bin/mkbuilder create mode 120000 tools/mkbuilder/lib/bash diff --git a/.gitignore b/.gitignore index 8ebc30a..4be39ca 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,13 @@ !.gitignore !.editorconfig -build/ +builders/ node_modules/ tmp/ vendor/ + +*.log +*.tmp + +# default output file for 'nix build' +result diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash new file mode 100644 index 0000000..b6ad481 --- /dev/null +++ b/tools/_common/lib/bash/prelude.bash @@ -0,0 +1,33 @@ +# This file should be sourced, not run + +set -o errexit +set -o nounset +set -o pipefail + +function warn { echo "$@" >&2; } +function die { warn "$@"; exit 1; } + +ASPIREBUILD_ORIG_PWD=$PWD + +ASPIREBUILD_TOOL_DIR=${ASPIREBUILD_TOOL_DIR:-$(dirname "$0")/..} +ASPIREBUILD_TOOL_DIR=$(realpath "$ASPIREBUILD_TOOL_DIR") + +ASPIREBUILD_BASE_DIR=${ASPIREBUILD_BASE_DIR:-$ASPIREBUILD_TOOL_DIR/../..} +ASPIREBUILD_BASE_DIR=$(realpath "$ASPIREBUILD_BASE_DIR") + +ASPIREBUILD_BUILDER_DIR=${ASPIREBUILD_BUILDER_DIR:-$ASPIREBUILD_BASE_DIR/builders} +ASPIREBUILD_BUILDER_DIR=$(realpath "$ASPIREBUILD_BUILDER_DIR") + +export ASPIREBUILD_ORIG_PWD ASPIREBUILD_TOOL_DIR ASPIREBUILD_BASE_DIR ASPIREBUILD_BUILDER_DIR + +# We bail out early if our cwd contains spaces, rather than risk stepping on this mine later. +# We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. +cd "$ASPIREBUILD_TOOL_DIR" || die "Could not cd to $ASPIREBUILD_TOOL_DIR" +[[ "$PWD" =~ [[:space:]] ]] && die "Refusing to deal with working directory containing whitespace. Aborted." + +mkdir -p "$ASPIREBUILD_BUILDER_DIR" + +for file in $(shopt -s nullglob; echo "$ASPIREBUILD_TOOL_DIR/lib/bash/prelude.d"/*.bash); do + # shellcheck source=/dev/null + source "$file" +done diff --git a/tools/mkbuilder/bin/builder-info b/tools/mkbuilder/bin/builder-info new file mode 100755 index 0000000..7ebafb7 --- /dev/null +++ b/tools/mkbuilder/bin/builder-info @@ -0,0 +1,6 @@ +#!/bin/bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +env | grep ASPIREBUILD_ diff --git a/tools/mkbuilder/bin/mkbuilder b/tools/mkbuilder/bin/mkbuilder new file mode 100755 index 0000000..f51d10c --- /dev/null +++ b/tools/mkbuilder/bin/mkbuilder @@ -0,0 +1,13 @@ +#!/bin/bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +cd $ASPIREBUILD_BASE_DIR || die "could not chdir to $ASPIREBUILD_BASE_DIR" + +build_dir=${1:-$(mktemp -d $ASPIREBUILD_BUILDER_DIR/builder.XXXXXXXXXX)} + +[[ $build_dir =~ ^/ ]] || build_dir=$ASPIREBUILD_BUILDER_DIR/$build_dir + +# This does the trick, but we really want a nix build... +git clone . $build_dir diff --git a/tools/mkbuilder/lib/bash b/tools/mkbuilder/lib/bash new file mode 120000 index 0000000..2c630ca --- /dev/null +++ b/tools/mkbuilder/lib/bash @@ -0,0 +1 @@ +../../_common/lib/bash \ No newline at end of file From f3558337603af33c17c5530b04f52815e46cd100 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 15:37:10 -0700 Subject: [PATCH 14/41] add prelude.d/reset.bash to set PATH only to /nix/* dirs Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.d/reset.bash | 13 +++++++++++++ tools/mkbuilder/bin/builder-info | 3 ++- tools/mkbuilder/bin/mkbuilder | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tools/_common/lib/bash/prelude.d/reset.bash diff --git a/tools/_common/lib/bash/prelude.d/reset.bash b/tools/_common/lib/bash/prelude.d/reset.bash new file mode 100644 index 0000000..fdfa853 --- /dev/null +++ b/tools/_common/lib/bash/prelude.d/reset.bash @@ -0,0 +1,13 @@ +# This file should be sourced, not run +# +# WARNING: By no means does this script create a fully hermetic environment. +# It's meant to help make builds more reproducible, but does not guarantee it. + +function nix_only_path { + perl -MEnv=PATH -E 'say join ":", sort (grep /^\/nix/, (split /:/, $PATH));' +} + +# note: __ORIG_PATH is not exported by default +__ORIG_PATH=$PATH + +PATH=$(nix_only_path) diff --git a/tools/mkbuilder/bin/builder-info b/tools/mkbuilder/bin/builder-info index 7ebafb7..580a230 100755 --- a/tools/mkbuilder/bin/builder-info +++ b/tools/mkbuilder/bin/builder-info @@ -3,4 +3,5 @@ # shellcheck disable=SC2086 source "$(dirname $0)"/../lib/bash/prelude.bash -env | grep ASPIREBUILD_ +env | grep -E '^ASPIREBUILD_' +echo "PATH=$PATH" diff --git a/tools/mkbuilder/bin/mkbuilder b/tools/mkbuilder/bin/mkbuilder index f51d10c..d59a9fe 100755 --- a/tools/mkbuilder/bin/mkbuilder +++ b/tools/mkbuilder/bin/mkbuilder @@ -10,4 +10,6 @@ build_dir=${1:-$(mktemp -d $ASPIREBUILD_BUILDER_DIR/builder.XXXXXXXXXX)} [[ $build_dir =~ ^/ ]] || build_dir=$ASPIREBUILD_BUILDER_DIR/$build_dir # This does the trick, but we really want a nix build... -git clone . $build_dir +git clone . $build_dir > /dev/null + +echo $build_dir From 54f6548ad53408f2b6167dd3f34b95c4a4dbdadc Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 15:37:50 -0700 Subject: [PATCH 15/41] add missing git dependency Signed-off-by: Chuck Adams --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index e21f113..2561b12 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,7 @@ bash coreutils curl + git gnutar jq just From 24b684207bce2296198c852b311d8178a99a2fc7 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 17:04:40 -0700 Subject: [PATCH 16/41] add liberal comments to docker section of flake.nix Signed-off-by: Chuck Adams --- flake.nix | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2561b12..94d3c3e 100644 --- a/flake.nix +++ b/flake.nix @@ -34,17 +34,45 @@ }; # WIP docker support - # https://community.flake.parts/haskell-flake/docker + # https://nixos.org/manual/nixpkgs/stable/#ssec-pkgs-dockerTools-buildImage + # https://ryantm.github.io/nixpkgs/builders/images/dockertools/ (older but not hideous) packages = { dockerImage = pkgs.dockerTools.buildImage { name = "aspirebuild"; tag = builtins.substring 0 9 (self.rev or "dev"); # tag with git revision, or 'dev' if dirty + + # created = "now"; # defaults to 1 (that is, 1 second past the epoch). 'now' is not binary-reproducible + + config = { + # https://community.flake.parts/haskell-flake/docker Env = [ "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" "SYSTEM_CERTIFICATE_PATH=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" ]; + # Cmd = [ "/path/to/command" ]; + # WorkingDir = "/data"; + # Volumes = { "/data" = { }; }; }; + + # tag = "sometag" # default null = nix output hash + # + # fromImage = "/path/to/repository-tarball.tgz"; # default null ~ 'FROM scratch' + # fromImageName = null; # default null = first image in repo tarball + # fromImageTag = "latest"; # default null = first tag for the base image + # diskSize = 1024; # default 1024 (always in MiB) + # buildVMMemorySize = 512; # default 512 (always in MiB) + # + # copyToRoot = pkgs.buildEnv { + # name = "image-root"; + # paths = [ pkgs.redis ]; + # pathsToLink = [ "/bin" ]; + # }; + # + # runAsRoot = '' + # #!${pkgs.runtimeShell} + # mkdir -p /data + # ''; }; }; } From 8e747f8d0566a042d59b56b751654b75215e45ae Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 17:16:50 -0700 Subject: [PATCH 17/41] use /usr/bin/env in bash shebangs Signed-off-by: Chuck Adams --- tools/mkbuilder/bin/builder-info | 2 +- tools/mkbuilder/bin/mkbuilder | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mkbuilder/bin/builder-info b/tools/mkbuilder/bin/builder-info index 580a230..abe56e5 100755 --- a/tools/mkbuilder/bin/builder-info +++ b/tools/mkbuilder/bin/builder-info @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # shellcheck disable=SC2086 source "$(dirname $0)"/../lib/bash/prelude.bash diff --git a/tools/mkbuilder/bin/mkbuilder b/tools/mkbuilder/bin/mkbuilder index d59a9fe..2cd8af6 100755 --- a/tools/mkbuilder/bin/mkbuilder +++ b/tools/mkbuilder/bin/mkbuilder @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # shellcheck disable=SC2086 source "$(dirname $0)"/../lib/bash/prelude.bash From a41a9154ec88d5f0ca6515ec93095ecf0c38eba5 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 17:36:37 -0700 Subject: [PATCH 18/41] zap unnecessary || die checks after cd Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.bash | 9 +++++---- tools/legacy-aspiresync/bin/archive-plugin | 2 +- tools/mkbuilder/bin/mkbuilder | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash index b6ad481..e0d2292 100644 --- a/tools/_common/lib/bash/prelude.bash +++ b/tools/_common/lib/bash/prelude.bash @@ -4,10 +4,11 @@ set -o errexit set -o nounset set -o pipefail -function warn { echo "$@" >&2; } -function die { warn "$@"; exit 1; } +function warn { echo "$@" >&2; } +function die { warn "$@"; exit 1; } -ASPIREBUILD_ORIG_PWD=$PWD +# not exported by default +__ORIG_PWD=$PWD ASPIREBUILD_TOOL_DIR=${ASPIREBUILD_TOOL_DIR:-$(dirname "$0")/..} ASPIREBUILD_TOOL_DIR=$(realpath "$ASPIREBUILD_TOOL_DIR") @@ -18,7 +19,7 @@ ASPIREBUILD_BASE_DIR=$(realpath "$ASPIREBUILD_BASE_DIR") ASPIREBUILD_BUILDER_DIR=${ASPIREBUILD_BUILDER_DIR:-$ASPIREBUILD_BASE_DIR/builders} ASPIREBUILD_BUILDER_DIR=$(realpath "$ASPIREBUILD_BUILDER_DIR") -export ASPIREBUILD_ORIG_PWD ASPIREBUILD_TOOL_DIR ASPIREBUILD_BASE_DIR ASPIREBUILD_BUILDER_DIR +export ASPIREBUILD_TOOL_DIR ASPIREBUILD_BASE_DIR ASPIREBUILD_BUILDER_DIR # We bail out early if our cwd contains spaces, rather than risk stepping on this mine later. # We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. diff --git a/tools/legacy-aspiresync/bin/archive-plugin b/tools/legacy-aspiresync/bin/archive-plugin index b470153..7f2c208 100755 --- a/tools/legacy-aspiresync/bin/archive-plugin +++ b/tools/legacy-aspiresync/bin/archive-plugin @@ -46,7 +46,7 @@ function main() { exit 0 fi - cd "$slug" || die "could not chdir to $(pwd)/$slug" + cd "$slug" svn update --set-depth=immediates --revision "$rev" tags diff --git a/tools/mkbuilder/bin/mkbuilder b/tools/mkbuilder/bin/mkbuilder index 2cd8af6..96a4bc0 100755 --- a/tools/mkbuilder/bin/mkbuilder +++ b/tools/mkbuilder/bin/mkbuilder @@ -3,7 +3,7 @@ # shellcheck disable=SC2086 source "$(dirname $0)"/../lib/bash/prelude.bash -cd $ASPIREBUILD_BASE_DIR || die "could not chdir to $ASPIREBUILD_BASE_DIR" +cd $ASPIREBUILD_BASE_DIR build_dir=${1:-$(mktemp -d $ASPIREBUILD_BUILDER_DIR/builder.XXXXXXXXXX)} From d700939292ec8fdb426f29631ae7288e807a0a6f Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 18:11:41 -0700 Subject: [PATCH 19/41] add in-new-builder script Signed-off-by: Chuck Adams --- tools/mkbuilder/bin/in-new-builder | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 tools/mkbuilder/bin/in-new-builder diff --git a/tools/mkbuilder/bin/in-new-builder b/tools/mkbuilder/bin/in-new-builder new file mode 100755 index 0000000..d4a318c --- /dev/null +++ b/tools/mkbuilder/bin/in-new-builder @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +here=$(dirname "$0") +source "$here/../lib/bash/prelude.bash" + +[[ -n "$*" ]] || die "usage: $0 " + +builder=$(bin/mkbuilder) +[[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT + +cd "$builder" +"$@" From ae23eb2a2758dbca862bd11b19b79eca759199f2 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 18:23:47 -0700 Subject: [PATCH 20/41] clear out ASPIREBUILD_* vars before executing in in-new-builder Signed-off-by: Chuck Adams --- tools/mkbuilder/bin/in-new-builder | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/mkbuilder/bin/in-new-builder b/tools/mkbuilder/bin/in-new-builder index d4a318c..e86eff2 100755 --- a/tools/mkbuilder/bin/in-new-builder +++ b/tools/mkbuilder/bin/in-new-builder @@ -8,5 +8,10 @@ source "$here/../lib/bash/prelude.bash" builder=$(bin/mkbuilder) [[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT +# will take their default values inside the builder when invoking any tool +unset ASPIREBUILD_BASE_DIR +unset ASPIREBUILD_TOOL_DIR +unset ASPIREBUILD_BUILDER_DIR + cd "$builder" "$@" From afbb1ceecfc1a22a2df44eff38b2119becb399b0 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 18:32:27 -0700 Subject: [PATCH 21/41] symlink _common/lib to mkbuilder/lib Signed-off-by: Chuck Adams --- tools/mkbuilder/lib | 1 + tools/mkbuilder/lib/bash | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 tools/mkbuilder/lib delete mode 120000 tools/mkbuilder/lib/bash diff --git a/tools/mkbuilder/lib b/tools/mkbuilder/lib new file mode 120000 index 0000000..7108a4a --- /dev/null +++ b/tools/mkbuilder/lib @@ -0,0 +1 @@ +../_common/lib \ No newline at end of file diff --git a/tools/mkbuilder/lib/bash b/tools/mkbuilder/lib/bash deleted file mode 120000 index 2c630ca..0000000 --- a/tools/mkbuilder/lib/bash +++ /dev/null @@ -1 +0,0 @@ -../../_common/lib/bash \ No newline at end of file From 7a711d964d638161816446dff3e27f2f05d14687 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 18:53:25 -0700 Subject: [PATCH 22/41] renae mkbuilder -> builder Signed-off-by: Chuck Adams --- tools/{mkbuilder => builder}/bin/builder-info | 0 tools/{mkbuilder => builder}/bin/in-new-builder | 0 tools/{mkbuilder => builder}/bin/mkbuilder | 0 tools/{mkbuilder => builder}/lib | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tools/{mkbuilder => builder}/bin/builder-info (100%) rename tools/{mkbuilder => builder}/bin/in-new-builder (100%) rename tools/{mkbuilder => builder}/bin/mkbuilder (100%) rename tools/{mkbuilder => builder}/lib (100%) diff --git a/tools/mkbuilder/bin/builder-info b/tools/builder/bin/builder-info similarity index 100% rename from tools/mkbuilder/bin/builder-info rename to tools/builder/bin/builder-info diff --git a/tools/mkbuilder/bin/in-new-builder b/tools/builder/bin/in-new-builder similarity index 100% rename from tools/mkbuilder/bin/in-new-builder rename to tools/builder/bin/in-new-builder diff --git a/tools/mkbuilder/bin/mkbuilder b/tools/builder/bin/mkbuilder similarity index 100% rename from tools/mkbuilder/bin/mkbuilder rename to tools/builder/bin/mkbuilder diff --git a/tools/mkbuilder/lib b/tools/builder/lib similarity index 100% rename from tools/mkbuilder/lib rename to tools/builder/lib From ce1d831f820e76370754f4bb20a24e00fe8735fa Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Sun, 2 Nov 2025 19:07:00 -0700 Subject: [PATCH 23/41] rename in-new-builder -> with-new-builder Signed-off-by: Chuck Adams --- tools/builder/bin/{in-new-builder => with-new-builder} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/builder/bin/{in-new-builder => with-new-builder} (100%) diff --git a/tools/builder/bin/in-new-builder b/tools/builder/bin/with-new-builder similarity index 100% rename from tools/builder/bin/in-new-builder rename to tools/builder/bin/with-new-builder From 92ea77f3ff2e321d77379daf2c41abc1434ff5f2 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 3 Nov 2025 18:38:40 -0700 Subject: [PATCH 24/41] use single ASPIREBUILD for root, clone to .aspirebuild subdir Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.bash | 45 ++++++++++++++++++----------- tools/builder/bin/mkbuilder | 12 ++++---- tools/builder/bin/with-new-builder | 6 ++-- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash index e0d2292..18e34c6 100644 --- a/tools/_common/lib/bash/prelude.bash +++ b/tools/_common/lib/bash/prelude.bash @@ -4,31 +4,42 @@ set -o errexit set -o nounset set -o pipefail -function warn { echo "$@" >&2; } -function die { warn "$@"; exit 1; } - -# not exported by default +# These are not exported, but will be visible in the tool if they wish to do so. __ORIG_PWD=$PWD +__HERE=$(dirname "$0") -ASPIREBUILD_TOOL_DIR=${ASPIREBUILD_TOOL_DIR:-$(dirname "$0")/..} -ASPIREBUILD_TOOL_DIR=$(realpath "$ASPIREBUILD_TOOL_DIR") - -ASPIREBUILD_BASE_DIR=${ASPIREBUILD_BASE_DIR:-$ASPIREBUILD_TOOL_DIR/../..} -ASPIREBUILD_BASE_DIR=$(realpath "$ASPIREBUILD_BASE_DIR") - -ASPIREBUILD_BUILDER_DIR=${ASPIREBUILD_BUILDER_DIR:-$ASPIREBUILD_BASE_DIR/builders} -ASPIREBUILD_BUILDER_DIR=$(realpath "$ASPIREBUILD_BUILDER_DIR") +function warn { echo "$@" >&2; } +function die { warn "$@"; exit 1; } -export ASPIREBUILD_TOOL_DIR ASPIREBUILD_BASE_DIR ASPIREBUILD_BUILDER_DIR +function _find_aspirebuild_root { + local dir + dir=$(dirname "$0") + dir=$(realpath "$dir") + while [[ -n $dir ]]; do + if [[ -d "$dir/tools/_common" ]]; then + echo "$dir" + return + fi + newdir=$(dirname "$dir") + [[ $newdir = "$dir" ]] && break + dir=$newdir + done + die "Could not find aspirebuild root in any parent directory of $PWD" +} + +# spawned builders will set this to the new builder's .aspirebuild dir +export ASPIREBUILD=${ASPIREBUILD:-$(_find_aspirebuild_root)} +export ASPIREBUILD_DEPTH=$(( ${ASPIREBUILD_DEPTH:--1} + 1 )) # base is level 0, meaning builders will be at level 1 + +[[ $ASPIREBUILD_DEPTH -lt ${ASPIREBUILD_RECURSION_LIMIT:-10} ]] || die "Maximum aspirebuild recursion depth reached. Aborted." # We bail out early if our cwd contains spaces, rather than risk stepping on this mine later. # We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. -cd "$ASPIREBUILD_TOOL_DIR" || die "Could not cd to $ASPIREBUILD_TOOL_DIR" -[[ "$PWD" =~ [[:space:]] ]] && die "Refusing to deal with working directory containing whitespace. Aborted." +[[ "$ASPIREBUILD" =~ [[:space:]] ]] && die "Refusing to deal with working directory containing whitespace. Aborted." -mkdir -p "$ASPIREBUILD_BUILDER_DIR" +cd "$ASPIREBUILD" -for file in $(shopt -s nullglob; echo "$ASPIREBUILD_TOOL_DIR/lib/bash/prelude.d"/*.bash); do +for file in $(shopt -s nullglob; echo "$__HERE/../lib/bash/prelude.d"/*.bash); do # shellcheck source=/dev/null source "$file" done diff --git a/tools/builder/bin/mkbuilder b/tools/builder/bin/mkbuilder index 96a4bc0..d8452fa 100755 --- a/tools/builder/bin/mkbuilder +++ b/tools/builder/bin/mkbuilder @@ -3,13 +3,13 @@ # shellcheck disable=SC2086 source "$(dirname $0)"/../lib/bash/prelude.bash -cd $ASPIREBUILD_BASE_DIR +builders=${ASPIREBUILD_BUILDER_DIR:-$ASPIREBUILD/builders} +mkdir -p "$builders" -build_dir=${1:-$(mktemp -d $ASPIREBUILD_BUILDER_DIR/builder.XXXXXXXXXX)} +builder=${1:-$(mktemp -d $builders/builder.XXXXXXXXXX)} -[[ $build_dir =~ ^/ ]] || build_dir=$ASPIREBUILD_BUILDER_DIR/$build_dir +[[ $builder =~ ^/ ]] || builder=$builders/$builder -# This does the trick, but we really want a nix build... -git clone . $build_dir > /dev/null +git clone . $builder/.aspirebuild > /dev/null -echo $build_dir +echo $builder diff --git a/tools/builder/bin/with-new-builder b/tools/builder/bin/with-new-builder index e86eff2..5bc8a62 100755 --- a/tools/builder/bin/with-new-builder +++ b/tools/builder/bin/with-new-builder @@ -5,13 +5,11 @@ source "$here/../lib/bash/prelude.bash" [[ -n "$*" ]] || die "usage: $0 " -builder=$(bin/mkbuilder) +builder=$("$__HERE/mkbuilder") [[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT # will take their default values inside the builder when invoking any tool -unset ASPIREBUILD_BASE_DIR -unset ASPIREBUILD_TOOL_DIR -unset ASPIREBUILD_BUILDER_DIR +export ASPIREBUILD=$builder/.aspirebuild cd "$builder" "$@" From 0a22a6ed9093552c16e8b3cca633ede7d4433ef2 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 3 Nov 2025 18:45:35 -0700 Subject: [PATCH 25/41] use absolute path for __HERE Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash index 18e34c6..a336d75 100644 --- a/tools/_common/lib/bash/prelude.bash +++ b/tools/_common/lib/bash/prelude.bash @@ -7,6 +7,7 @@ set -o pipefail # These are not exported, but will be visible in the tool if they wish to do so. __ORIG_PWD=$PWD __HERE=$(dirname "$0") +__HERE=$(realpath "$__HERE") function warn { echo "$@" >&2; } function die { warn "$@"; exit 1; } From ea7655455756af588f0335387005c0ae04574f85 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 3 Nov 2025 19:47:14 -0700 Subject: [PATCH 26/41] more flake fiddling Signed-off-by: Chuck Adams --- flake.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 94d3c3e..c6b385e 100644 --- a/flake.nix +++ b/flake.nix @@ -12,9 +12,10 @@ let pkgs = nixpkgs.legacyPackages.${system}; in { + # devShell is great for 'nix develop', but running bash in a builder still uses noninteractive bash :-/ devShell = with pkgs; mkShell { buildInputs = [ - bash + bashInteractive coreutils curl git @@ -27,10 +28,15 @@ php84Packages.composer subversion sqlite + systemfd tzdata + watchexec zip zstd ]; + shellHook = '' + export SHELL=${lib.getExe pkgs.bash} + ''; }; # WIP docker support From bbe6da5592c63a6aa2390be0fee95f21575e1ca5 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 3 Nov 2025 19:47:41 -0700 Subject: [PATCH 27/41] zap update-plugin-trunk Signed-off-by: Chuck Adams --- tools/builder/bin/with-new-builder | 1 - .../legacy-aspiresync/bin/update-plugin-trunk | 100 ------------------ 2 files changed, 101 deletions(-) delete mode 100755 tools/legacy-aspiresync/bin/update-plugin-trunk diff --git a/tools/builder/bin/with-new-builder b/tools/builder/bin/with-new-builder index 5bc8a62..e9ed6cc 100755 --- a/tools/builder/bin/with-new-builder +++ b/tools/builder/bin/with-new-builder @@ -8,7 +8,6 @@ source "$here/../lib/bash/prelude.bash" builder=$("$__HERE/mkbuilder") [[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT -# will take their default values inside the builder when invoking any tool export ASPIREBUILD=$builder/.aspirebuild cd "$builder" diff --git a/tools/legacy-aspiresync/bin/update-plugin-trunk b/tools/legacy-aspiresync/bin/update-plugin-trunk deleted file mode 100755 index d569656..0000000 --- a/tools/legacy-aspiresync/bin/update-plugin-trunk +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - - -. $(dirname $0)/prelude.bash -cd $PLUGINS_DIR - -enforce_svn_root - -[[ -n ${NO_SVN_CLEANUP:-} ]] || svn cleanup - -repo=$(basename $(pwd)) -mkdir -p $ARCHIVE_DIR/$repo/trunks/ - -# These have so many redundant files (usually from recursively nested tags) that they're effectively zip bombs -bombs='all-in-one-contact-buttons-wpshare247|biblesupersearch' - -# These produce errors when svn attempts to update them -corrupt='a2-optimized-wp|better-links|countdown-timer|facebook-album-sync|font-awesome-the-easy-way|up-wp-cart' - -# combined blacklist -BLACKLIST="^($bombs|$corrupt)\$" - -#### - -function main() { - for dir in $* - do - process $dir - done -} - -function process() { - local dir=$1 - - echo -e "\nPROCESSING: $dir [trunk]" - - if ! [[ $dir =~ ^[-A-Za-z0-9]+$ ]]; then - finalize $dir "malformed dir ignored" - return - fi - - if [[ $dir =~ $BLACKLIST ]]; then - finalize $dir "blacklisted dir skipped" - return - fi - - [[ -d $dir ]] || svn update --ignore-externals --set-depth=empty $dir - - # Note: using a new /trunks base directory, but still using .trunk suffix so archive files can co-mingle - local rev=$(get_revision $dir) - local tar=$ARCHIVE_DIR/$repo/trunks/$rev/$rev.$dir.trunk.tar - local ztar=$tar.zst - - if [[ -f $ztar ]]; then - finalize $dir "archive exists at revision $rev" - return - fi - - mkdir -p $(dirname $tar) - update $dir - archive $dir $ztar - ls -lh $ztar - finalize $dir "archived at revision $rev" -} - -function update() { - local dir=$1 - svn update --set-depth=immediates --force $dir/tags # the list of versions is lightweight and useful metadata - svn update --ignore-externals --set-depth=infinity --force $dir/trunk -} - -function archive() { - local dir=$1 - local ztar=$2 - - tar cf - ./$dir | zstd --progress > $ztar.tmp - mv $ztar.tmp $ztar -} - -function finalize() { - local dir=$1 - local message=$2 - - warn "$dir: $message" - [[ -n ${UPDATE_KEEP_DIRS:-} ]] || rm -rf "$dir" -} - -function get_revision() { - local dir=$1 - local rev=$(svn info -r HEAD --show-item last-changed-revision $dir) - if [[ -z $rev ]]; then - die "fatal could not retrieve revision for $dir" - fi - echo $rev -} - -################ - -main $* - From 710b5c14a4cde618fa1be67db15bbe528c69fa87 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Tue, 4 Nov 2025 11:59:26 -0700 Subject: [PATCH 28/41] update algorithm for finding/creating $ASPIRESYNC Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.bash | 56 +++++++++++++++++++++-------- tools/legacy-aspiresync/lib | 1 + 2 files changed, 43 insertions(+), 14 deletions(-) create mode 120000 tools/legacy-aspiresync/lib diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash index a336d75..08399b6 100644 --- a/tools/_common/lib/bash/prelude.bash +++ b/tools/_common/lib/bash/prelude.bash @@ -1,5 +1,18 @@ # This file should be sourced, not run +# +# Relevant Environment Variables: +# +# ASPIREBUILD absolute path to current builder's .aspirebuild directory or symlink +# In a root builder, the .aspirebuild link points to the absolute path of the root. +# +# ASPIREBUILD_DEPTH current depth of recursive builders +# +# ASPIREBUILD_DEPTH_LIMIT maximum depth of recursive builders (default 10, including the root) +# +# ASPIREBUILD_BUILDER_DIR absolute path to location of new builders. defaults to $ASPIREBUILD/builders +# + set -o errexit set -o nounset set -o pipefail @@ -7,40 +20,55 @@ set -o pipefail # These are not exported, but will be visible in the tool if they wish to do so. __ORIG_PWD=$PWD __HERE=$(dirname "$0") -__HERE=$(realpath "$__HERE") +__HERE=$(realpath -s "$__HERE") # canonicalize only, don't resolve symlinks function warn { echo "$@" >&2; } function die { warn "$@"; exit 1; } -function _find_aspirebuild_root { +function _find_or_create_dot_aspirebuild { local dir dir=$(dirname "$0") - dir=$(realpath "$dir") + dir=$(realpath -s "$dir") # like $__HERE, in that we don't chase symlinks while [[ -n $dir ]]; do - if [[ -d "$dir/tools/_common" ]]; then - echo "$dir" + local dot_aspirebuild="$dir/.aspirebuild" + + if [[ -d "$dot_aspirebuild" ]]; then + dir=$(realpath "$dir") + echo "$dir/.aspirebuild" return + elif [[ -d "$dir/tools/_common/lib" ]]; then + dir=$(realpath "$dir") + ln -s "$dir" "$dot_aspirebuild" + echo "$dir/.aspirebuild" + return + else + newdir=$(dirname "$dir") + [[ $newdir = "$dir" ]] && break + dir=$newdir fi - newdir=$(dirname "$dir") - [[ $newdir = "$dir" ]] && break - dir=$newdir done die "Could not find aspirebuild root in any parent directory of $PWD" } # spawned builders will set this to the new builder's .aspirebuild dir -export ASPIREBUILD=${ASPIREBUILD:-$(_find_aspirebuild_root)} +export ASPIREBUILD=${ASPIREBUILD:-$(_find_or_create_dot_aspirebuild)} export ASPIREBUILD_DEPTH=$(( ${ASPIREBUILD_DEPTH:--1} + 1 )) # base is level 0, meaning builders will be at level 1 -[[ $ASPIREBUILD_DEPTH -lt ${ASPIREBUILD_RECURSION_LIMIT:-10} ]] || die "Maximum aspirebuild recursion depth reached. Aborted." +[[ $ASPIREBUILD_DEPTH -lt ${ASPIREBUILD_DEPTH_LIMIT:-10} ]] || die "Maximum aspirebuild recursion depth reached. Aborted." -# We bail out early if our cwd contains spaces, rather than risk stepping on this mine later. +# We bail out early if our working directory contains spaces, rather than risk stepping on this mine later. # We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. -[[ "$ASPIREBUILD" =~ [[:space:]] ]] && die "Refusing to deal with working directory containing whitespace. Aborted." +[[ "$ASPIREBUILD" =~ [[:space:]] ]] && die "Refusing to deal with aspirebuild directory containing whitespace. Aborted." cd "$ASPIREBUILD" -for file in $(shopt -s nullglob; echo "$__HERE/../lib/bash/prelude.d"/*.bash); do +# Run all prelude files under the current tool's lib (usually symlinked to tools/_common/lib) +preludes=$( + shopt -s nullglob + echo "$__HERE"/../lib/bash/prelude.d/*.bash "$__HERE"/../local/lib/bash/prelude.d/*.bash +); + +for file in $preludes; do # shellcheck source=/dev/null - source "$file" + [[ -f $file ]] && source "$file" done diff --git a/tools/legacy-aspiresync/lib b/tools/legacy-aspiresync/lib new file mode 120000 index 0000000..7108a4a --- /dev/null +++ b/tools/legacy-aspiresync/lib @@ -0,0 +1 @@ +../_common/lib \ No newline at end of file From 803de1c67d832a37826cffce4994b7f7b6853ddf Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Tue, 4 Nov 2025 12:38:06 -0700 Subject: [PATCH 29/41] add init-builder-flake but keep it manual for now Signed-off-by: Chuck Adams --- tools/builder/bin/init-builder-flake | 16 ++++++++++++++++ tools/builder/bin/mkbuilder | 4 ++++ 2 files changed, 20 insertions(+) create mode 100755 tools/builder/bin/init-builder-flake diff --git a/tools/builder/bin/init-builder-flake b/tools/builder/bin/init-builder-flake new file mode 100755 index 0000000..5e8ad17 --- /dev/null +++ b/tools/builder/bin/init-builder-flake @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +builder=${1:-} +[[ -n $builder ]] || die "Usage: $0 builder-dir" +[[ -d $builder ]] || die "$0: $builder: no such directory" +[[ -d $builder/.aspirebuild/builder ]] || die "$0: $builder does not look like a builder" + +[[ -f $builder/flake.nix ]] || cp $builder/.aspirebuild/flake.nix $builder +[[ -f $builder/flake.lock ]] || cp $builder/.aspirebuild/flake.lock $builder +[[ -e $builder/.envrc ]] || echo "use flake" > $builder/.envrc + +# I really hate this part +# [[ -d $builder/.git ]] || ( cd $builder; git -c advice.addEmbeddedRepo=false init; git -c advice.addEmbeddedRepo=false add . ) >/dev/null diff --git a/tools/builder/bin/mkbuilder b/tools/builder/bin/mkbuilder index d8452fa..cfb15a1 100755 --- a/tools/builder/bin/mkbuilder +++ b/tools/builder/bin/mkbuilder @@ -11,5 +11,9 @@ builder=${1:-$(mktemp -d $builders/builder.XXXXXXXXXX)} [[ $builder =~ ^/ ]] || builder=$builders/$builder git clone . $builder/.aspirebuild > /dev/null +ln -s $builder $builder/.aspirebuild/builder + +# should be deferred til later, e.g. in a Justfile, when it doesn't have to be silenced +# $__HERE/init-builder-flake $builder >/dev/null 2>&1 echo $builder From ba512081c387d5ba7a6399c6a45930e304621891 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Tue, 4 Nov 2025 12:50:33 -0700 Subject: [PATCH 30/41] add default justfile Signed-off-by: Chuck Adams --- justfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 justfile diff --git a/justfile b/justfile new file mode 100644 index 0000000..2ec8f5b --- /dev/null +++ b/justfile @@ -0,0 +1,8 @@ + +builder := `readlink -f $ASPIREBUILD/builder` + +nix-develop: nix-init + cd {{builder}} && nix develop + +nix-init: + tools/builder/bin/init-builder-flake {{builder}} From d13837584502f19f9167c53348b198aa66075ee7 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Tue, 4 Nov 2025 13:01:19 -0700 Subject: [PATCH 31/41] use path:. for builder flake in justfile Signed-off-by: Chuck Adams --- justfile | 6 +++++- tools/builder/bin/init-builder-flake | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 2ec8f5b..4f4d346 100644 --- a/justfile +++ b/justfile @@ -1,8 +1,12 @@ builder := `readlink -f $ASPIREBUILD/builder` +alias nix-shell := nix-develop + +[group: 'nix'] nix-develop: nix-init - cd {{builder}} && nix develop + cd {{builder}} && nix develop path:. +[group: 'nix'] nix-init: tools/builder/bin/init-builder-flake {{builder}} diff --git a/tools/builder/bin/init-builder-flake b/tools/builder/bin/init-builder-flake index 5e8ad17..09fffd2 100755 --- a/tools/builder/bin/init-builder-flake +++ b/tools/builder/bin/init-builder-flake @@ -11,6 +11,3 @@ builder=${1:-} [[ -f $builder/flake.nix ]] || cp $builder/.aspirebuild/flake.nix $builder [[ -f $builder/flake.lock ]] || cp $builder/.aspirebuild/flake.lock $builder [[ -e $builder/.envrc ]] || echo "use flake" > $builder/.envrc - -# I really hate this part -# [[ -d $builder/.git ]] || ( cd $builder; git -c advice.addEmbeddedRepo=false init; git -c advice.addEmbeddedRepo=false add . ) >/dev/null From f1032d5c982ffe2f8998fd4c1a1e70afe2d1e0ce Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Tue, 4 Nov 2025 14:13:45 -0700 Subject: [PATCH 32/41] add php-init to justfile Signed-off-by: Chuck Adams --- justfile | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index 4f4d346..c81dcd7 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,19 @@ +# AspireBuild justfile -builder := `readlink -f $ASPIREBUILD/builder` +builder := if env('ASPIREBUILD', '') != '' { + `readlink -f $ASPIREBUILD/builder` +} else { + justfile_directory() +} + +#### General targets + +init: php-init + +php-init: + cd {{builder}} && composer install + +#### Nix targets (only in builders, not allowed in root) alias nix-shell := nix-develop @@ -8,5 +22,15 @@ nix-develop: nix-init cd {{builder}} && nix develop path:. [group: 'nix'] -nix-init: +nix-init: _require-builder tools/builder/bin/init-builder-flake {{builder}} + + +#### Utility targets + +@_require-builder: + if ! [[ -d {{builder}}/.aspirebuild/builder ]]; then \ + echo '{{style("error")}}This command can only be run in a builder, not the AspireBuild root{{NORMAL}}' >&2; \ + exit 1; \ + fi + From 7e9d32bbf89bc6d4ec2bd070bf8cc79052282ce3 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 10 Nov 2025 09:43:10 -0700 Subject: [PATCH 33/41] switch to flake-parts Signed-off-by: Chuck Adams --- flake.lock | 59 ++++++++++++++-------------- flake.nix | 110 ++++++++++++++++++++++++----------------------------- 2 files changed, 79 insertions(+), 90 deletions(-) diff --git a/flake.lock b/flake.lock index e79ec35..41a8ed7 100644 --- a/flake.lock +++ b/flake.lock @@ -1,60 +1,59 @@ { "nodes": { - "flake-utils": { + "flake-parts": { "inputs": { - "systems": "systems" + "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "lastModified": 1762440070, + "narHash": "sha256-xxdepIcb39UJ94+YydGP221rjnpkDZUlykKuF54PsqI=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "26d05891e14c88eb4a5d5bee659c0db5afb609d8", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "owner": "hercules-ci", + "repo": "flake-parts", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1761864852, - "narHash": "sha256-Wmpt+cRwQ8O5r9gGmkls2xOSBwqs1Og5qMWrREHTRv4=", - "owner": "nixos", + "lastModified": 1762596750, + "narHash": "sha256-rXXuz51Bq7DHBlfIjN7jO8Bu3du5TV+3DSADBX7/9YQ=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a5e47a4bea3996a6511f1da3cf6ba92e71a95f04", + "rev": "b6a8526db03f735b89dd5ff348f53f752e7ddc8e", "type": "github" }, "original": { - "owner": "nixos", + "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", - "rev": "a5e47a4bea3996a6511f1da3cf6ba92e71a95f04", "type": "github" } }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { + "nixpkgs-lib": { "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "lastModified": 1761765539, + "narHash": "sha256-b0yj6kfvO8ApcSE+QmA6mUfu8IYG6/uU28OFn4PaC8M=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "719359f4562934ae99f5443f20aa06c2ffff91fc", "type": "github" }, "original": { - "owner": "nix-systems", - "repo": "default", + "owner": "nix-community", + "repo": "nixpkgs.lib", "type": "github" } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index c6b385e..08f93e2 100644 --- a/flake.nix +++ b/flake.nix @@ -1,20 +1,45 @@ { + # a work in progress. only devshell is supported for now. + + description = "AspireBuild"; + inputs = { -# nixpkgs.url = "github:nixos/nixpkgs"; -# nixpkgs.url = "github:nixos/nixpkgs/25.05"; -# nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; - nixpkgs.url = "github:nixos/nixpkgs/a5e47a4bea3996a6511f1da3cf6ba92e71a95f04"; # (2025-10-30) - flake-utils.url = "github:numtide/flake-utils/11707dc2f618dd54ca8739b309ec4fc024de578b"; # (2024-11-13) + flake-parts.url = "github:hercules-ci/flake-parts"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; }; - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgs = nixpkgs.legacyPackages.${system}; - in { - # devShell is great for 'nix develop', but running bash in a builder still uses noninteractive bash :-/ - devShell = with pkgs; mkShell { - buildInputs = [ + outputs = + inputs@{ flake-parts, self, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + # To import an internal flake module: ./other.nix + # To import an external flake module: + # 1. Add foo to inputs + # 2. Add foo as a parameter to the outputs function + # 3. Add here: foo.flakeModule + ]; + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + + # most of the flake should go in here + perSystem = + { + config, + self', + inputs', + pkgs, + system, + ... + }: + # Per-system attributes can be defined here. The self' and inputs' + # module parameters provide easy access to attributes of the same + # system. + let + buildInputs = with pkgs; [ bashInteractive coreutils curl @@ -34,53 +59,18 @@ zip zstd ]; - shellHook = '' - export SHELL=${lib.getExe pkgs.bash} - ''; - }; - - # WIP docker support - # https://nixos.org/manual/nixpkgs/stable/#ssec-pkgs-dockerTools-buildImage - # https://ryantm.github.io/nixpkgs/builders/images/dockertools/ (older but not hideous) - packages = { - dockerImage = pkgs.dockerTools.buildImage { - name = "aspirebuild"; - tag = builtins.substring 0 9 (self.rev or "dev"); # tag with git revision, or 'dev' if dirty - - # created = "now"; # defaults to 1 (that is, 1 second past the epoch). 'now' is not binary-reproducible - - - config = { - # https://community.flake.parts/haskell-flake/docker - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - "SYSTEM_CERTIFICATE_PATH=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - ]; - # Cmd = [ "/path/to/command" ]; - # WorkingDir = "/data"; - # Volumes = { "/data" = { }; }; - }; - - # tag = "sometag" # default null = nix output hash - # - # fromImage = "/path/to/repository-tarball.tgz"; # default null ~ 'FROM scratch' - # fromImageName = null; # default null = first image in repo tarball - # fromImageTag = "latest"; # default null = first tag for the base image - # diskSize = 1024; # default 1024 (always in MiB) - # buildVMMemorySize = 512; # default 512 (always in MiB) - # - # copyToRoot = pkgs.buildEnv { - # name = "image-root"; - # paths = [ pkgs.redis ]; - # pathsToLink = [ "/bin" ]; - # }; - # - # runAsRoot = '' - # #!${pkgs.runtimeShell} - # mkdir -p /data - # ''; + in + { + devShells.default = pkgs.mkShell { + inherit buildInputs; }; + + # invoke with `nix fmt flake.nix` + formatter = pkgs.nixfmt-rfc-style; }; - } - ); + + flake = { + # system-agnostic flake attributes go here. we don't have any yet. + }; + }; } From a1c86b6c1a4770ad9fc74bc065d998f3e07e3659 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 10 Nov 2025 11:13:27 -0700 Subject: [PATCH 34/41] use flake-root to set ASPIREBUILD Signed-off-by: Chuck Adams --- flake.lock | 16 ++++++++++++++++ flake.nix | 15 +++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 41a8ed7..126144e 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,21 @@ "type": "github" } }, + "flake-root": { + "locked": { + "lastModified": 1723604017, + "narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=", + "owner": "srid", + "repo": "flake-root", + "rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1762596750, @@ -52,6 +67,7 @@ "root": { "inputs": { "flake-parts": "flake-parts", + "flake-root": "flake-root", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 08f93e2..2cd7fd8 100644 --- a/flake.nix +++ b/flake.nix @@ -5,18 +5,15 @@ inputs = { flake-parts.url = "github:hercules-ci/flake-parts"; + flake-root.url = "github:srid/flake-root"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; }; outputs = - inputs@{ flake-parts, self, ... }: + inputs@{ self, flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } { imports = [ - # To import an internal flake module: ./other.nix - # To import an external flake module: - # 1. Add foo to inputs - # 2. Add foo as a parameter to the outputs function - # 3. Add here: foo.flakeModule + inputs.flake-root.flakeModule ]; systems = [ "x86_64-linux" @@ -63,6 +60,12 @@ { devShells.default = pkgs.mkShell { inherit buildInputs; + + inputsFrom = [ config.flake-root.devShell ]; # sets $FLAKE_ROOT + + shellHook = '' + export ASPIREBUILD=$FLAKE_ROOT + ''; }; # invoke with `nix fmt flake.nix` From 02eb27467b510f31d0da82b99935a8ef7c421760 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 10 Nov 2025 12:21:35 -0700 Subject: [PATCH 35/41] init builders with template Signed-off-by: Chuck Adams --- tools/_common/lib/bash/prelude.bash | 47 ++----------------- .../lib/bash/prelude.d/nix-only-path.bash | 10 ++++ tools/_common/lib/bash/prelude.d/reset.bash | 13 ----- tools/builder/bin/builder-info | 7 --- tools/builder/bin/init-builder | 28 +++++++++++ tools/builder/bin/init-builder-flake | 13 ----- tools/builder/bin/{mkbuilder => make-builder} | 6 +-- .../{with-new-builder => with-temp-builder} | 4 +- tools/builder/templates/default/_meta/install | 11 +++++ tools/builder/templates/default/justfile | 3 ++ 10 files changed, 57 insertions(+), 85 deletions(-) create mode 100644 tools/_common/lib/bash/prelude.d/nix-only-path.bash delete mode 100644 tools/_common/lib/bash/prelude.d/reset.bash delete mode 100755 tools/builder/bin/builder-info create mode 100755 tools/builder/bin/init-builder delete mode 100755 tools/builder/bin/init-builder-flake rename tools/builder/bin/{mkbuilder => make-builder} (56%) rename tools/builder/bin/{with-new-builder => with-temp-builder} (74%) create mode 100755 tools/builder/templates/default/_meta/install create mode 100644 tools/builder/templates/default/justfile diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash index 08399b6..fd41d92 100644 --- a/tools/_common/lib/bash/prelude.bash +++ b/tools/_common/lib/bash/prelude.bash @@ -1,23 +1,10 @@ # This file should be sourced, not run -# -# Relevant Environment Variables: -# -# ASPIREBUILD absolute path to current builder's .aspirebuild directory or symlink -# In a root builder, the .aspirebuild link points to the absolute path of the root. -# -# ASPIREBUILD_DEPTH current depth of recursive builders -# -# ASPIREBUILD_DEPTH_LIMIT maximum depth of recursive builders (default 10, including the root) -# -# ASPIREBUILD_BUILDER_DIR absolute path to location of new builders. defaults to $ASPIREBUILD/builders -# - set -o errexit set -o nounset set -o pipefail -# These are not exported, but will be visible in the tool if they wish to do so. +# These are not exported, but will be visible in the tool's script if they need them __ORIG_PWD=$PWD __HERE=$(dirname "$0") __HERE=$(realpath -s "$__HERE") # canonicalize only, don't resolve symlinks @@ -25,36 +12,8 @@ __HERE=$(realpath -s "$__HERE") # canonicalize only, don't resolve symlinks function warn { echo "$@" >&2; } function die { warn "$@"; exit 1; } -function _find_or_create_dot_aspirebuild { - local dir - dir=$(dirname "$0") - dir=$(realpath -s "$dir") # like $__HERE, in that we don't chase symlinks - while [[ -n $dir ]]; do - local dot_aspirebuild="$dir/.aspirebuild" - - if [[ -d "$dot_aspirebuild" ]]; then - dir=$(realpath "$dir") - echo "$dir/.aspirebuild" - return - elif [[ -d "$dir/tools/_common/lib" ]]; then - dir=$(realpath "$dir") - ln -s "$dir" "$dot_aspirebuild" - echo "$dir/.aspirebuild" - return - else - newdir=$(dirname "$dir") - [[ $newdir = "$dir" ]] && break - dir=$newdir - fi - done - die "Could not find aspirebuild root in any parent directory of $PWD" -} - -# spawned builders will set this to the new builder's .aspirebuild dir -export ASPIREBUILD=${ASPIREBUILD:-$(_find_or_create_dot_aspirebuild)} -export ASPIREBUILD_DEPTH=$(( ${ASPIREBUILD_DEPTH:--1} + 1 )) # base is level 0, meaning builders will be at level 1 - -[[ $ASPIREBUILD_DEPTH -lt ${ASPIREBUILD_DEPTH_LIMIT:-10} ]] || die "Maximum aspirebuild recursion depth reached. Aborted." +# This is set in flake.nix, so this will only be blank if not in a nix environment. +[[ -n $ASPIREBUILD ]] || die "ASPIREBUILD environment variable not set. Please set it to the absolute path of an AspireBuild git repo." # We bail out early if our working directory contains spaces, rather than risk stepping on this mine later. # We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. diff --git a/tools/_common/lib/bash/prelude.d/nix-only-path.bash b/tools/_common/lib/bash/prelude.d/nix-only-path.bash new file mode 100644 index 0000000..20bbcb6 --- /dev/null +++ b/tools/_common/lib/bash/prelude.d/nix-only-path.bash @@ -0,0 +1,10 @@ +# This file should be sourced, not run +# +# Sets PATH to include only directories under /nix +# +# By no means does this script create a fully hermetic environment. +# It's meant to help make builds more reproducible, but does not guarantee it. + +__ORIG_PATH=$PATH + +PATH=$(awk -v RS=: -v ORS=: '$0 ~ /^\/nix\/.*/' <<<"$PATH") diff --git a/tools/_common/lib/bash/prelude.d/reset.bash b/tools/_common/lib/bash/prelude.d/reset.bash deleted file mode 100644 index fdfa853..0000000 --- a/tools/_common/lib/bash/prelude.d/reset.bash +++ /dev/null @@ -1,13 +0,0 @@ -# This file should be sourced, not run -# -# WARNING: By no means does this script create a fully hermetic environment. -# It's meant to help make builds more reproducible, but does not guarantee it. - -function nix_only_path { - perl -MEnv=PATH -E 'say join ":", sort (grep /^\/nix/, (split /:/, $PATH));' -} - -# note: __ORIG_PATH is not exported by default -__ORIG_PATH=$PATH - -PATH=$(nix_only_path) diff --git a/tools/builder/bin/builder-info b/tools/builder/bin/builder-info deleted file mode 100755 index abe56e5..0000000 --- a/tools/builder/bin/builder-info +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC2086 -source "$(dirname $0)"/../lib/bash/prelude.bash - -env | grep -E '^ASPIREBUILD_' -echo "PATH=$PATH" diff --git a/tools/builder/bin/init-builder b/tools/builder/bin/init-builder new file mode 100755 index 0000000..d7bca74 --- /dev/null +++ b/tools/builder/bin/init-builder @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +[[ -n "$*" ]] || die "usage: $0 " + +builder=$1 +shift + +cd $builder + +# TODO: make this an option --template=foo (I'm not doing option parsing in bash) +template=${BUILDER_TEMPLATE:-default} + +# A template containing a hash, e.g. .#foo is assumed to be a nix expression +if [[ $template =~ [#] ]]; then + nix flake init -t $template + exit 0 +fi + +[[ $template =~ ^/ ]] || template=$(realpath -s "$__HERE/../templates/$template") + +[[ -d $template ]] || die "No such directory: $template " + +[[ -f $template/_meta/install ]] || die "Template $template does not contain a _meta/install script" + +$template/_meta/install "$template" "$builder" diff --git a/tools/builder/bin/init-builder-flake b/tools/builder/bin/init-builder-flake deleted file mode 100755 index 09fffd2..0000000 --- a/tools/builder/bin/init-builder-flake +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC2086 -source "$(dirname $0)"/../lib/bash/prelude.bash - -builder=${1:-} -[[ -n $builder ]] || die "Usage: $0 builder-dir" -[[ -d $builder ]] || die "$0: $builder: no such directory" -[[ -d $builder/.aspirebuild/builder ]] || die "$0: $builder does not look like a builder" - -[[ -f $builder/flake.nix ]] || cp $builder/.aspirebuild/flake.nix $builder -[[ -f $builder/flake.lock ]] || cp $builder/.aspirebuild/flake.lock $builder -[[ -e $builder/.envrc ]] || echo "use flake" > $builder/.envrc diff --git a/tools/builder/bin/mkbuilder b/tools/builder/bin/make-builder similarity index 56% rename from tools/builder/bin/mkbuilder rename to tools/builder/bin/make-builder index cfb15a1..580fde4 100755 --- a/tools/builder/bin/mkbuilder +++ b/tools/builder/bin/make-builder @@ -10,10 +10,6 @@ builder=${1:-$(mktemp -d $builders/builder.XXXXXXXXXX)} [[ $builder =~ ^/ ]] || builder=$builders/$builder -git clone . $builder/.aspirebuild > /dev/null -ln -s $builder $builder/.aspirebuild/builder - -# should be deferred til later, e.g. in a Justfile, when it doesn't have to be silenced -# $__HERE/init-builder-flake $builder >/dev/null 2>&1 +$__HERE/init-builder $builder >/dev/null echo $builder diff --git a/tools/builder/bin/with-new-builder b/tools/builder/bin/with-temp-builder similarity index 74% rename from tools/builder/bin/with-new-builder rename to tools/builder/bin/with-temp-builder index e9ed6cc..6318bc4 100755 --- a/tools/builder/bin/with-new-builder +++ b/tools/builder/bin/with-temp-builder @@ -5,10 +5,8 @@ source "$here/../lib/bash/prelude.bash" [[ -n "$*" ]] || die "usage: $0 " -builder=$("$__HERE/mkbuilder") +builder=$("$__HERE/make-builder") [[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT -export ASPIREBUILD=$builder/.aspirebuild - cd "$builder" "$@" diff --git a/tools/builder/templates/default/_meta/install b/tools/builder/templates/default/_meta/install new file mode 100755 index 0000000..8d523c3 --- /dev/null +++ b/tools/builder/templates/default/_meta/install @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +template=$1 +dest=$2 + +[[ -f $template/justfile ]] && cp "$template/justfile" "$dest" || true + +mkdir -p "$dest/in" +mkdir -p "$dest/out" + + diff --git a/tools/builder/templates/default/justfile b/tools/builder/templates/default/justfile new file mode 100644 index 0000000..cf6b834 --- /dev/null +++ b/tools/builder/templates/default/justfile @@ -0,0 +1,3 @@ + +@builder-env: + echo "ASPIREBUILD=$ASPIREBUILD" From 2b9144bcd4b8e01320c76178835320ad53d691e2 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Mon, 10 Nov 2025 13:07:16 -0700 Subject: [PATCH 36/41] remove useless root justfile Signed-off-by: Chuck Adams --- justfile | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 justfile diff --git a/justfile b/justfile deleted file mode 100644 index c81dcd7..0000000 --- a/justfile +++ /dev/null @@ -1,36 +0,0 @@ -# AspireBuild justfile - -builder := if env('ASPIREBUILD', '') != '' { - `readlink -f $ASPIREBUILD/builder` -} else { - justfile_directory() -} - -#### General targets - -init: php-init - -php-init: - cd {{builder}} && composer install - -#### Nix targets (only in builders, not allowed in root) - -alias nix-shell := nix-develop - -[group: 'nix'] -nix-develop: nix-init - cd {{builder}} && nix develop path:. - -[group: 'nix'] -nix-init: _require-builder - tools/builder/bin/init-builder-flake {{builder}} - - -#### Utility targets - -@_require-builder: - if ! [[ -d {{builder}}/.aspirebuild/builder ]]; then \ - echo '{{style("error")}}This command can only be run in a builder, not the AspireBuild root{{NORMAL}}' >&2; \ - exit 1; \ - fi - From a223a95da7302fa26fda384e33944342fa4c2f25 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Wed, 12 Nov 2025 09:20:05 -0700 Subject: [PATCH 37/41] add ffi extension to flake.nix Signed-off-by: Chuck Adams --- flake.nix | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 2cd7fd8..8f328da 100644 --- a/flake.nix +++ b/flake.nix @@ -32,9 +32,6 @@ system, ... }: - # Per-system attributes can be defined here. The self' and inputs' - # module parameters provide easy access to attributes of the same - # system. let buildInputs = with pkgs; [ bashInteractive @@ -47,6 +44,7 @@ lrzip perl php + php84Extensions.ffi php84Packages.composer subversion sqlite @@ -65,9 +63,25 @@ shellHook = '' export ASPIREBUILD=$FLAKE_ROOT + export PHP_INI_SCAN_DIR=:${self'.packages.default}/.php-ini ''; }; + packages.default = pkgs.stdenv.mkDerivation { + inherit buildInputs; + + name = "aspirebuild"; + + buildPhase = '' + mkdir -p $out/.php-ini + + echo "extension=${pkgs.php84Extensions.ffi}/lib/php/extensions/ffi.so" > $out/.php-ini/php.ini + ''; + + installPhase = "true"; # if installPhase is absent or blank, it tries to run 'just' for some reason + src = ./.; + }; + # invoke with `nix fmt flake.nix` formatter = pkgs.nixfmt-rfc-style; }; From fee146d0b7d68d1eb3c5bea35a70909a06a7919a Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Wed, 12 Nov 2025 09:55:56 -0700 Subject: [PATCH 38/41] add intl extension to flake Signed-off-by: Chuck Adams --- flake.nix | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 8f328da..0371c38 100644 --- a/flake.nix +++ b/flake.nix @@ -12,9 +12,8 @@ outputs = inputs@{ self, flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } { - imports = [ - inputs.flake-root.flakeModule - ]; + imports = [ inputs.flake-root.flakeModule ]; + systems = [ "x86_64-linux" "aarch64-linux" @@ -30,6 +29,7 @@ inputs', pkgs, system, + lib, ... }: let @@ -45,6 +45,7 @@ perl php php84Extensions.ffi + php84Extensions.intl php84Packages.composer subversion sqlite @@ -65,6 +66,9 @@ export ASPIREBUILD=$FLAKE_ROOT export PHP_INI_SCAN_DIR=:${self'.packages.default}/.php-ini ''; + + # in case $FLAKE_ROOT isn't available, this should also work + # export ASPIREBUILD=$(${lib.getExe config.flake-root.package}) }; packages.default = pkgs.stdenv.mkDerivation { @@ -72,14 +76,16 @@ name = "aspirebuild"; + src = ./.; + buildPhase = '' mkdir -p $out/.php-ini echo "extension=${pkgs.php84Extensions.ffi}/lib/php/extensions/ffi.so" > $out/.php-ini/php.ini + echo "extension=${pkgs.php84Extensions.intl}/lib/php/extensions/intl.so" > $out/.php-ini/php.ini ''; - installPhase = "true"; # if installPhase is absent or blank, it tries to run 'just' for some reason - src = ./.; + installPhase = "true"; # if installPhase is absent or blank, it defaults to 'just' for some reason }; # invoke with `nix fmt flake.nix` From 710f23ee94c112419286910a15267162b5a8b964 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Wed, 12 Nov 2025 11:34:59 -0700 Subject: [PATCH 39/41] build all needed extensions Signed-off-by: Chuck Adams --- flake.nix | 71 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 0371c38..18cf59c 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - # a work in progress. only devshell is supported for now. + # a work in progress. description = "AspireBuild"; @@ -44,8 +44,6 @@ lrzip perl php - php84Extensions.ffi - php84Extensions.intl php84Packages.composer subversion sqlite @@ -55,6 +53,53 @@ zip zstd ]; + + extensions = with pkgs.php84Extensions; [ + bcmath + curl + ffi + filter + gettext + gmp + intl + mbstring + pdo + pdo_sqlite + readline + sockets + sodium + sqlite3 + + # full list in /nix/store/nsybw5k5jcqwccbgslfq5psmqh3x3svs-php-with-extensions-8.4.14/lib/php.ini + + # future consideration + #ctype + #dom + #fileinfo + #iconv + #mysqli + #mysqlnd + #openssl + #pcntl + #pdo_mysql + #pdo_pgsql + #pgsql + #posix + #session + #simplexml + #sockets + #tokenizer + #xmlreader + #xmlwriter + #zip + #zlib + ]; + + zend-extensions = with pkgs.php84Extensions; [ + opcache + xdebug + ]; + in { devShells.default = pkgs.mkShell { @@ -64,7 +109,8 @@ shellHook = '' export ASPIREBUILD=$FLAKE_ROOT - export PHP_INI_SCAN_DIR=:${self'.packages.default}/.php-ini + export SELF_DIR=${self'.packages.default} + export PHP_INI_SCAN_DIR=:${self'.packages.default} ''; # in case $FLAKE_ROOT isn't available, this should also work @@ -78,11 +124,22 @@ src = ./.; + php-ini = + let + get-extension-name = name: builtins.elemAt (builtins.split "-" name) 2; # "php-intl-8.4.13" -> "intl" + ext-line = ext: "extension = ${ext}/lib/php/extensions/${get-extension-name ext.name}.so"; + zend-ext-line = ext: "zend_extension = ${ext}/lib/php/extensions/${get-extension-name ext.name}.so"; + in + (map ext-line extensions) ++ (map zend-ext-line zend-extensions); + buildPhase = '' - mkdir -p $out/.php-ini + mkdir -p $out + + cat << EOF > $out/php.ini + ; PHP extensions for AspireBuild - echo "extension=${pkgs.php84Extensions.ffi}/lib/php/extensions/ffi.so" > $out/.php-ini/php.ini - echo "extension=${pkgs.php84Extensions.intl}/lib/php/extensions/intl.so" > $out/.php-ini/php.ini + ${lib.concatStringsSep "\n" config.packages.default.php-ini} + EOF ''; installPhase = "true"; # if installPhase is absent or blank, it defaults to 'just' for some reason From 41176df617e662f87b2e5d86dc25deabcba35ff0 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Thu, 13 Nov 2025 11:43:49 -0700 Subject: [PATCH 40/41] document The One Flake Signed-off-by: Chuck Adams --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 18cf59c..9151083 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,7 @@ { - # a work in progress. + # This flake, which I'm now calling The One Flake, describes the entire AspireBuild monorepo including all its tools. + # The eventual goal is to make each tool its own flake, as well as each builder, with each tool and builder depending + # on the AspireBuild flake. However we're not there yet, so currently we manage everything through The One Flake. description = "AspireBuild"; From 48f3d161036276a7086f58b264387167468feea5 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Thu, 13 Nov 2025 12:30:00 -0700 Subject: [PATCH 41/41] use dontInstall = true; add note about $ASPIREBUILD's impurity Signed-off-by: Chuck Adams --- flake.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 9151083..256b1fb 100644 --- a/flake.nix +++ b/flake.nix @@ -115,8 +115,14 @@ export PHP_INI_SCAN_DIR=:${self'.packages.default} ''; - # in case $FLAKE_ROOT isn't available, this should also work - # export ASPIREBUILD=$(${lib.getExe config.flake-root.package}) + # in case $FLAKE_ROOT isn't available, this should also work. + # + # export ASPIREBUILD=$(${lib.getExe config.flake-root.package}) + # + # Note that the flake root is impure state, and thus should never be set in an attribute or in a file, + # so technically this makes our flake impure because of all of the dependencies on $ASPIREBUILD. But since + # $ASPIREBUILD in the package always points into the nix store for this flake, we can get away with it + # while still allowing local dev to point to the working copy. }; packages.default = pkgs.stdenv.mkDerivation { @@ -144,7 +150,7 @@ EOF ''; - installPhase = "true"; # if installPhase is absent or blank, it defaults to 'just' for some reason + dontInstall = true; }; # invoke with `nix fmt flake.nix`