diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c2f271c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false + +[*.{nix,yml,yaml}] +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4be39ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# ignore all dotfiles and dotdirs by default, with exceptions below +.* +!.gitignore +!.editorconfig + +builders/ +node_modules/ +tmp/ +vendor/ + +*.log +*.tmp + +# default output file for 'nix build' +result diff --git a/README.md b/README.md index 4d9c4f2..ceffe52 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ # aspirebuild + +## Requirements + +* Nix, with flakes support. I _highly_ recommend the [Determinate Systems installer](https://docs.determinate.systems/) +* Recommended: `direnv` (should be available on apt/brew/dnf/pacman/etc) + +## Quick Start + +``` +echo "use flake" > .envrc && direnv allow # if using direnv +nix develop +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..126144e --- /dev/null +++ b/flake.lock @@ -0,0 +1,77 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1762440070, + "narHash": "sha256-xxdepIcb39UJ94+YydGP221rjnpkDZUlykKuF54PsqI=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "26d05891e14c88eb4a5d5bee659c0db5afb609d8", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-root": { + "locked": { + "lastModified": 1723604017, + "narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=", + "owner": "srid", + "repo": "flake-root", + "rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1762596750, + "narHash": "sha256-rXXuz51Bq7DHBlfIjN7jO8Bu3du5TV+3DSADBX7/9YQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b6a8526db03f735b89dd5ff348f53f752e7ddc8e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1761765539, + "narHash": "sha256-b0yj6kfvO8ApcSE+QmA6mUfu8IYG6/uU28OFn4PaC8M=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "719359f4562934ae99f5443f20aa06c2ffff91fc", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "flake-root": "flake-root", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..256b1fb --- /dev/null +++ b/flake.nix @@ -0,0 +1,164 @@ +{ + # This flake, which I'm now calling The One Flake, describes the entire AspireBuild monorepo including all its tools. + # The eventual goal is to make each tool its own flake, as well as each builder, with each tool and builder depending + # on the AspireBuild flake. However we're not there yet, so currently we manage everything through The One Flake. + + description = "AspireBuild"; + + inputs = { + flake-parts.url = "github:hercules-ci/flake-parts"; + flake-root.url = "github:srid/flake-root"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = + inputs@{ self, flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ inputs.flake-root.flakeModule ]; + + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + + # most of the flake should go in here + perSystem = + { + config, + self', + inputs', + pkgs, + system, + lib, + ... + }: + let + buildInputs = with pkgs; [ + bashInteractive + coreutils + curl + git + gnutar + jq + just + lrzip + perl + php + php84Packages.composer + subversion + sqlite + systemfd + tzdata + watchexec + zip + zstd + ]; + + extensions = with pkgs.php84Extensions; [ + bcmath + curl + ffi + filter + gettext + gmp + intl + mbstring + pdo + pdo_sqlite + readline + sockets + sodium + sqlite3 + + # full list in /nix/store/nsybw5k5jcqwccbgslfq5psmqh3x3svs-php-with-extensions-8.4.14/lib/php.ini + + # future consideration + #ctype + #dom + #fileinfo + #iconv + #mysqli + #mysqlnd + #openssl + #pcntl + #pdo_mysql + #pdo_pgsql + #pgsql + #posix + #session + #simplexml + #sockets + #tokenizer + #xmlreader + #xmlwriter + #zip + #zlib + ]; + + zend-extensions = with pkgs.php84Extensions; [ + opcache + xdebug + ]; + + in + { + devShells.default = pkgs.mkShell { + inherit buildInputs; + + inputsFrom = [ config.flake-root.devShell ]; # sets $FLAKE_ROOT + + shellHook = '' + export ASPIREBUILD=$FLAKE_ROOT + export SELF_DIR=${self'.packages.default} + export PHP_INI_SCAN_DIR=:${self'.packages.default} + ''; + + # in case $FLAKE_ROOT isn't available, this should also work. + # + # export ASPIREBUILD=$(${lib.getExe config.flake-root.package}) + # + # Note that the flake root is impure state, and thus should never be set in an attribute or in a file, + # so technically this makes our flake impure because of all of the dependencies on $ASPIREBUILD. But since + # $ASPIREBUILD in the package always points into the nix store for this flake, we can get away with it + # while still allowing local dev to point to the working copy. + }; + + packages.default = pkgs.stdenv.mkDerivation { + inherit buildInputs; + + name = "aspirebuild"; + + src = ./.; + + php-ini = + let + get-extension-name = name: builtins.elemAt (builtins.split "-" name) 2; # "php-intl-8.4.13" -> "intl" + ext-line = ext: "extension = ${ext}/lib/php/extensions/${get-extension-name ext.name}.so"; + zend-ext-line = ext: "zend_extension = ${ext}/lib/php/extensions/${get-extension-name ext.name}.so"; + in + (map ext-line extensions) ++ (map zend-ext-line zend-extensions); + + buildPhase = '' + mkdir -p $out + + cat << EOF > $out/php.ini + ; PHP extensions for AspireBuild + + ${lib.concatStringsSep "\n" config.packages.default.php-ini} + EOF + ''; + + dontInstall = true; + }; + + # invoke with `nix fmt flake.nix` + formatter = pkgs.nixfmt-rfc-style; + }; + + flake = { + # system-agnostic flake attributes go here. we don't have any yet. + }; + }; +} diff --git a/tools/_common/lib/bash/prelude.bash b/tools/_common/lib/bash/prelude.bash new file mode 100644 index 0000000..fd41d92 --- /dev/null +++ b/tools/_common/lib/bash/prelude.bash @@ -0,0 +1,33 @@ +# This file should be sourced, not run + +set -o errexit +set -o nounset +set -o pipefail + +# These are not exported, but will be visible in the tool's script if they need them +__ORIG_PWD=$PWD +__HERE=$(dirname "$0") +__HERE=$(realpath -s "$__HERE") # canonicalize only, don't resolve symlinks + +function warn { echo "$@" >&2; } +function die { warn "$@"; exit 1; } + +# This is set in flake.nix, so this will only be blank if not in a nix environment. +[[ -n $ASPIREBUILD ]] || die "ASPIREBUILD environment variable not set. Please set it to the absolute path of an AspireBuild git repo." + +# We bail out early if our working directory contains spaces, rather than risk stepping on this mine later. +# We make reasonable efforts to quote bash arguments, but 'bash' and 'reasonable' do not belong in the same sentence. +[[ "$ASPIREBUILD" =~ [[:space:]] ]] && die "Refusing to deal with aspirebuild directory containing whitespace. Aborted." + +cd "$ASPIREBUILD" + +# Run all prelude files under the current tool's lib (usually symlinked to tools/_common/lib) +preludes=$( + shopt -s nullglob + echo "$__HERE"/../lib/bash/prelude.d/*.bash "$__HERE"/../local/lib/bash/prelude.d/*.bash +); + +for file in $preludes; do + # shellcheck source=/dev/null + [[ -f $file ]] && source "$file" +done diff --git a/tools/_common/lib/bash/prelude.d/nix-only-path.bash b/tools/_common/lib/bash/prelude.d/nix-only-path.bash new file mode 100644 index 0000000..20bbcb6 --- /dev/null +++ b/tools/_common/lib/bash/prelude.d/nix-only-path.bash @@ -0,0 +1,10 @@ +# This file should be sourced, not run +# +# Sets PATH to include only directories under /nix +# +# By no means does this script create a fully hermetic environment. +# It's meant to help make builds more reproducible, but does not guarantee it. + +__ORIG_PATH=$PATH + +PATH=$(awk -v RS=: -v ORS=: '$0 ~ /^\/nix\/.*/' <<<"$PATH") diff --git a/tools/builder/bin/init-builder b/tools/builder/bin/init-builder new file mode 100755 index 0000000..d7bca74 --- /dev/null +++ b/tools/builder/bin/init-builder @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +[[ -n "$*" ]] || die "usage: $0 " + +builder=$1 +shift + +cd $builder + +# TODO: make this an option --template=foo (I'm not doing option parsing in bash) +template=${BUILDER_TEMPLATE:-default} + +# A template containing a hash, e.g. .#foo is assumed to be a nix expression +if [[ $template =~ [#] ]]; then + nix flake init -t $template + exit 0 +fi + +[[ $template =~ ^/ ]] || template=$(realpath -s "$__HERE/../templates/$template") + +[[ -d $template ]] || die "No such directory: $template " + +[[ -f $template/_meta/install ]] || die "Template $template does not contain a _meta/install script" + +$template/_meta/install "$template" "$builder" diff --git a/tools/builder/bin/make-builder b/tools/builder/bin/make-builder new file mode 100755 index 0000000..580fde4 --- /dev/null +++ b/tools/builder/bin/make-builder @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086 +source "$(dirname $0)"/../lib/bash/prelude.bash + +builders=${ASPIREBUILD_BUILDER_DIR:-$ASPIREBUILD/builders} +mkdir -p "$builders" + +builder=${1:-$(mktemp -d $builders/builder.XXXXXXXXXX)} + +[[ $builder =~ ^/ ]] || builder=$builders/$builder + +$__HERE/init-builder $builder >/dev/null + +echo $builder diff --git a/tools/builder/bin/with-temp-builder b/tools/builder/bin/with-temp-builder new file mode 100755 index 0000000..6318bc4 --- /dev/null +++ b/tools/builder/bin/with-temp-builder @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +here=$(dirname "$0") +source "$here/../lib/bash/prelude.bash" + +[[ -n "$*" ]] || die "usage: $0 " + +builder=$("$__HERE/make-builder") +[[ -z ${KEEP_BUILDER:-} ]] && trap "rm -rf $builder" EXIT + +cd "$builder" +"$@" diff --git a/tools/builder/lib b/tools/builder/lib new file mode 120000 index 0000000..7108a4a --- /dev/null +++ b/tools/builder/lib @@ -0,0 +1 @@ +../_common/lib \ No newline at end of file diff --git a/tools/builder/templates/default/_meta/install b/tools/builder/templates/default/_meta/install new file mode 100755 index 0000000..8d523c3 --- /dev/null +++ b/tools/builder/templates/default/_meta/install @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +template=$1 +dest=$2 + +[[ -f $template/justfile ]] && cp "$template/justfile" "$dest" || true + +mkdir -p "$dest/in" +mkdir -p "$dest/out" + + diff --git a/tools/builder/templates/default/justfile b/tools/builder/templates/default/justfile new file mode 100644 index 0000000..cf6b834 --- /dev/null +++ b/tools/builder/templates/default/justfile @@ -0,0 +1,3 @@ + +@builder-env: + echo "ASPIREBUILD=$ASPIREBUILD" diff --git a/tools/legacy-aspiresync/bin/archive-plugin b/tools/legacy-aspiresync/bin/archive-plugin new file mode 100755 index 0000000..7f2c208 --- /dev/null +++ b/tools/legacy-aspiresync/bin/archive-plugin @@ -0,0 +1,94 @@ +#!/bin/bash + +# shellcheck disable=SC2046 +. $(dirname "$0")/prelude.bash + +OUTPUT_DIR=$ARCHIVE_DIR/plugins/revisions +mkdir -p "$OUTPUT_DIR" + +# These have so many redundant files (usually from recursively nested tags) that they're effectively zip bombs +bombs='all-in-one-contact-buttons-wpshare247|biblesupersearch' + +# These produce errors when svn attempts to update them +corrupt='a2-optimized-wp|better-links|countdown-timer|facebook-album-sync|font-awesome-the-easy-way|up-wp-cart' + +# combined deny-list +DENYLIST="^($bombs|$corrupt)\$" + +#### + +function main() { + local slug=$1 + local tag=$2 + local rev=${3:-HEAD} + + echo "" + echo "PROCESSING: $slug [tag=$tag rev=$rev]" + + if ! [[ $slug =~ ^[-A-Za-z0-9]+$ ]]; then + die "malformed slug skipped: $slug" + return + fi + + if [[ $slug =~ $DENYLIST ]]; then + die "slug skipped due to matching explicit deny pattern: $slug" + return + fi + + enter_tmpdir + + svn co --depth=empty "$PLUGINS_REMOTE/$slug" + [[ $rev = 'HEAD' ]] && rev=$(get_revision "$slug") + local tar=$OUTPUT_DIR/$rev/$rev.$slug.$tag.tar.zst + + if [[ -f $tar ]] && [[ -z "${FORCE:-}" ]]; then + echo "archive exists for $slug [tag=$tag rev=$rev file=$tar]" + exit 0 + fi + + cd "$slug" + + svn update --set-depth=immediates --revision "$rev" tags + + if [[ $tag = 'trunk' ]]; then + svn update --set-depth=infinity --revision "$rev" trunk + else + [[ -d tags/$tag ]] || die "Could not find tag: $slug/tags/$tag" + svn update --set-depth=infinity --revision "$rev" "tags/$tag" + fi + + svn update --set-depth=infinity --revision "$rev" assets + + rm -rf .svn + cd .. + + mkdir -p $(dirname "$tar") + tar cf - "./$slug" | zstd --progress > "$tar.tmp" + mv "$tar.tmp" "$tar" + + echo "ARCHIVED: $slug [tag=$tag rev=$rev file=$tar]" +} + +function enter_tmpdir() { + local tmpdir + tmpdir=$(mktemp -d "$TMPDIR/svn.XXXXXXXX") + # shellcheck disable=SC2064 + trap "rm -rf $tmpdir" EXIT + cd "$tmpdir" || die "could not set pwd to $tmpdir" +} + +function get_revision() { + local dir=$1 + local rev + rev=$(svn info -r HEAD --show-item last-changed-revision "$dir") + + if [[ -z $rev ]]; then + die "fatal could not retrieve revision for $dir" + fi + echo "$rev" +} + +################ + +main "$@" + diff --git a/tools/legacy-aspiresync/bin/ls-updated-plugins b/tools/legacy-aspiresync/bin/ls-updated-plugins new file mode 100755 index 0000000..f0d396b --- /dev/null +++ b/tools/legacy-aspiresync/bin/ls-updated-plugins @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +. $(dirname $0)/prelude.bash + +date=$1 +[[ -n $date ]] || die "usage: $0 " + +checkout=$(mktemp -d $TMPDIR/plugins-svn.XXXXXXXX) +trap "rm -rf $checkout" EXIT + +RUN cd $checkout +RUN svn co https://plugins.svn.wordpress.org --depth empty > /dev/null +RUN $BASE_DIR/bin/svn-changed-slugs $date diff --git a/tools/legacy-aspiresync/bin/ls-updated-themes b/tools/legacy-aspiresync/bin/ls-updated-themes new file mode 100755 index 0000000..6cca519 --- /dev/null +++ b/tools/legacy-aspiresync/bin/ls-updated-themes @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +. $(dirname $0)/prelude.bash + +date=$1 +[[ -n $date ]] || die "usage: $0 " + +checkout=$(mktemp -d $TMPDIR/themes-svn.XXXXXXXX) +trap "rm -rf $checkout" EXIT + +RUN cd $checkout +RUN svn co https://themes.svn.wordpress.org --depth empty > /dev/null +RUN $BASE_DIR/bin/svn-changed-slugs $date diff --git a/tools/legacy-aspiresync/bin/prelude.bash b/tools/legacy-aspiresync/bin/prelude.bash new file mode 100644 index 0000000..e6d0b25 --- /dev/null +++ b/tools/legacy-aspiresync/bin/prelude.bash @@ -0,0 +1,68 @@ +# This file should be sourced, not run + +[[ -n $TRACE ]] && [[ $TRACE != 0 ]] && set -x + +set -o errexit +set -o nounset +set -o pipefail + +ORIG_PWD=$(pwd) +cd $(dirname $0)/.. +BASE_DIR=$(pwd) + +DATA_DIR=${DATA_DIR:-$HOME/svn-data} # should NOT be under the project root, it freaks IDEA out even if its excluded +ARCHIVE_DIR=${ARCHIVE_DIR:-$DATA_DIR/archive} + +TMPDIR=${TMPDIR:-/tmp} # no underscore on this one, it's an old unixism + +PLUGINS_REMOTE=${PLUGINS_REMOTE:-https://plugins.svn.wordpress.org} +PLUGINS_DIR=${PLUGINS_DIR:-$DATA_DIR/svn/plugins} + +THEMES_REMOTE=${THEMES_REMOTE:-https://themes.svn.wordpress.org} +THEMES_DIR=${THEMES_DIR:-$DATA_DIR/svn/themes} + +mkdir -p $DATA_DIR $ARCHIVE_DIR $PLUGINS_DIR $THEMES_DIR + +YMD=$(date +%Y-%m-%d) + +function warn { + echo "$@" >&2 +} + +function die() { + warn "$@" + exit 1 +} + +function RUN() { + local _run= + [[ -n ${DRY_RUN:-} ]] && [[ $DRY_RUN != 0 ]] && _run=echo + $_run "$@" +} + +function enforce_svn_root() { + [[ -d .svn ]] || die "$(pwd) does not look like a svn checkout -- exiting" +} + +function _use_checkout() { + type=$1 + remote=$2 + + cd $DATA_DIR + mkdir -p svn/$type + cd svn/$type + if [[ -d .svn ]]; then + $BASE_DIR/bin/svn-get-immediates + else + depth=${IMMEDIATES_DEPTH:-immediates} + svn checkout --ignore-externals --depth=$depth $remote + fi +} + +function use_plugins() { + _use_checkout plugins $PLUGINS_REMOTE +} + +function use_themes() { + _use_checkout themes $THEMES_REMOTE +} diff --git a/tools/legacy-aspiresync/bin/svn-changed-files b/tools/legacy-aspiresync/bin/svn-changed-files new file mode 100755 index 0000000..86e8e34 --- /dev/null +++ b/tools/legacy-aspiresync/bin/svn-changed-files @@ -0,0 +1,10 @@ +#!/bin/bash + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD +enforce_svn_root + +date=$1 +[[ -n $date ]] || die "Usage: $0 yyyy-mm-dd" + +RUN svn log --revision "{$date}:head" --verbose | perl -nE 'next unless m!^\s+[A-Z] (/.*)$/!; say $1' diff --git a/tools/legacy-aspiresync/bin/svn-changed-slugs b/tools/legacy-aspiresync/bin/svn-changed-slugs new file mode 100755 index 0000000..9cc4e15 --- /dev/null +++ b/tools/legacy-aspiresync/bin/svn-changed-slugs @@ -0,0 +1,11 @@ +#!/bin/bash + +. $(dirname $0)/prelude.bash +cd $ORIG_PWD +enforce_svn_root + +date=$1 +[[ -n $date ]] || die "Usage: $0 yyyy-mm-dd" + +RUN svn log --revision "{$date}:head" --verbose | perl -nE 'next unless m!^\s+[A-Z] /(.*?)/!; say $1 unless $seen{$1}++' + diff --git a/tools/legacy-aspiresync/lib b/tools/legacy-aspiresync/lib new file mode 120000 index 0000000..7108a4a --- /dev/null +++ b/tools/legacy-aspiresync/lib @@ -0,0 +1 @@ +../_common/lib \ No newline at end of file