diff --git a/nix/workbench/analyse/analyse.sh b/nix/workbench/analyse/analyse.sh index de857f950a7..f596fe32b2c 100644 --- a/nix/workbench/analyse/analyse.sh +++ b/nix/workbench/analyse/analyse.sh @@ -396,8 +396,8 @@ case "$op" in then v1=("${v0[@]/#logs/ 'unlog-db' --run-logs \"$adir\"/log-manifest-db.json }") else v1=("${v0[@]/#logs/ 'unlog' --run-logs \"$adir\"/log-manifest.json ${analysis_allowed_loanys[*]/#/--ok-loany } }") fi - v2=("${v1[@]/#read-context/ 'read-meta-genesis' --run-metafile \"$dir\"/meta.json --shelley-genesis \"$dir\"/genesis-shelley.json }") - v3=("${v2[@]/#write-context/ 'write-meta-genesis' --run-metafile \"$dir\"/meta.json --shelley-genesis \"$dir\"/genesis-shelley.json }") + v2=("${v1[@]/#read-context/ 'read-meta-genesis' --run-metafile \"$dir\"/meta.json --shelley-genesis \"$dir\"/genesis/genesis.shelley.json }") + v3=("${v2[@]/#write-context/ 'write-meta-genesis' --run-metafile \"$dir\"/meta.json --shelley-genesis \"$dir\"/genesis/genesis.shelley.json }") v4=("${v3[@]/#read-chain/ 'read-chain' --chain \"$adir\"/chain.json}") v5=("${v4[@]/#rebuild-chain/ 'rebuild-chain' ${filters[@]}}") v6=("${v5[@]/#dump-chain/ 'dump-chain' --chain \"$adir\"/chain.json --chain-rejecta \"$adir\"/chain-rejecta.json }") @@ -506,7 +506,7 @@ case "$op" in vi=(${vh[*]/#update/ 'compare' --ede nix/workbench/ede --report $adir/report-$run.org ${compares[*]} --template $adir/report-$run.ede }) vj=(${vi[*]/#multi-summary-json/ 'render-multi-summary' --json $adir/'summary.json' }) vk=(${vj[*]/#multi-summary-report/ 'render-multi-summary' --org-report $adir/'summary.org' }) - vl=(${vk[@]/#write-context/ 'write-meta-genesis' --run-metafile $dir/meta.json --shelley-genesis $dir/genesis-shelley.json }) + vl=(${vk[@]/#write-context/ 'write-meta-genesis' --run-metafile $dir/meta.json --shelley-genesis $dir/genesis/genesis.shelley.json }) local ops_final=(${vl[*]}) call_locli "$rtsmode" "${ops_final[@]}" diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index f99cadf919c..586a484bd2f 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -875,8 +875,14 @@ deploy-genesis-nomadcloud() { local genesis_file_name="${nomad_job_name}.tar.zst" msg "$(blue Creating) $(yellow "\"${genesis_file_name}\"") ..." # TODO: These files are link to file that don't exist! - rm "${dir}"/genesis/profile.json - rm "${dir}"/genesis/stake-delegator-keys + if test -e "${dir}"/genesis/profile.json + then + rm "${dir}"/genesis/profile.json + fi + if test -e "${dir}"/genesis/stake-delegators + then + rm "${dir}"/genesis/stake-delegators + fi find -L "${dir}"/genesis -printf "%P\n" \ | tar --create --zstd \ --dereference --hard-dereference \ diff --git a/nix/workbench/backend/runner.nix b/nix/workbench/backend/runner.nix index 34f33136ac0..ba5a757c34b 100644 --- a/nix/workbench/backend/runner.nix +++ b/nix/workbench/backend/runner.nix @@ -46,8 +46,9 @@ let export WB_BACKEND=${backendName} export WB_BACKEND_DATA=${backendDataDir} export WB_CREATE_TESTNET_DATA=''${WB_CREATE_TESTNET_DATA:-1} - export WB_DEPLOYMENT_NAME=''${WB_DEPLOYMENT_NAME:-$(basename $(pwd))} + export WB_GENESIS_CHUNKS=''${WB_GENESIS_CHUNKS:-0} export WB_MODULAR_GENESIS=''${WB_MODULAR_GENESIS:-0} + export WB_DEPLOYMENT_NAME=''${WB_DEPLOYMENT_NAME:-$(basename $(pwd))} export WB_LOCLI_DB=''${WB_LOCLI_DB:-1} if test -z "$(git status --porcelain --untracked-files=no)" then export WB_GITREV="$(git rev-parse HEAD)" diff --git a/nix/workbench/genesis/genesis-chunks.sh b/nix/workbench/genesis/genesis-chunks.sh new file mode 100644 index 00000000000..ba6dadc275f --- /dev/null +++ b/nix/workbench/genesis/genesis-chunks.sh @@ -0,0 +1,324 @@ +# shellcheck shell=bash +# +# Dataset-cached genesis backend. +# Used when WB_DATASET_GENESIS=1. +# +# Implements the backend interface: +# profile-cache-key-input-chunks, profile-cache-key-chunks, +# spec-chunks, pool-relays-chunks, +# genesis-create-chunks, +# derive-from-cache-chunks +# +# Three-level cache: +# +# Level 1: dataset cache ($cache_dir/genesis/dataset/) +# Keyed by dataset params (pools, delegators, supply, etc.) + hash. +# Shared across profiles with the same dataset. Expensive to create. +# Stores: +# - byron.dataset.json {bootStakeholders, heavyDelegation, nonAvvmBalances} +# - shelley.dataset.json {initialFunds, staking, maxLovelaceSupply} (can be 500MB+) +# - conway.dataset.json {initialDReps} (can be 200MB+) +# - byron-gen-command/ drep-keys/ pools-keys/ stake-delegators/ utxo-keys/ +# +# Level 2: protocol cache ($cache_dir/genesis/protocol/) +# Keyed by hash of genesis.shelley + genesis.alonzo + genesis.conway + +# genesis.dijkstra from the profile. Cheap to create. +# Stores the spec-derived parts WITHOUT dataset fields and WITHOUT +# systemStart (which is per-run): +# - byron.protocol.json (spec minus dataset fields) +# - shelley.protocol.json (spec minus dataset fields and systemStart) +# - alonzo.protocol.json (complete spec) +# - conway.protocol.json (spec minus dataset fields) +# - dijkstra.protocol.json (complete spec) +# +# Level 3: genesis cache ($cache_dir/genesis/) +# Keyed by: profile name + protocol hash + dataset hash. +# Assembled from level 1 + level 2 + systemStart using cat/sed/printf. +# No `jq` on any large file. No pretty-printing. + +# -- Backend interface --------------------------------------------------------- + +# The genesis cache key combines profile name, protocol params hash, and +# dataset hash. +profile-cache-key-input-chunks() { + local profile_json=$1 + local proto_hash dataset_hash + proto_hash=$(jq -c -S '{ + byron: ( .genesis.byron // {} ), + shelley: ( .genesis.shelley ), + alonzo: ( .genesis.alonzo ), + conway: ( .genesis.conway // {} ), + dijkstra: ( .genesis.dijkstra // {} ) + }' "$profile_json" | sha1sum | cut -c-7) + dataset_hash=$(dataset-cache-key "$profile_json" | sha1sum | cut -c-7) + echo "${proto_hash}-${dataset_hash}" +} + +profile-cache-key-chunks() { + local profile_json=$1 + local name + name=$(jq -r '.name' "$profile_json") + echo "${name}-$(profile-cache-key-input-chunks "$profile_json")" +} + +spec-chunks() { + spec-jq "$@"; +} + +pool-relays-chunks() { + pool-relays-jq "$@"; +} + +# Ensure sub-caches exist, assemble the genesis cache entry with symlinks +# to the dataset cache for key directories, and pre-assemble the genesis +# JSON files that don't change per run. Only genesis-shelley.json (which +# contains the per-run systemStart) is left for finalise-cache-entry-chunks. +genesis-create-chunks() { + local profile_json=$1 + local outdir=$2 + + mkdir -p "$outdir" + + local dataset_key dataset_entry proto_key proto_entry + dataset_key=$(dataset-cache-key "$profile_json") + protocol_key=$(protocol-cache-key "$profile_json") + # Resolve $outdir: $cache_dir/genesis/KEY as $cache_dir/genesis/{dataset|protocol}. + dataset_entry=$(dataset-cache-ensure "$profile_json" "$outdir/../dataset/$dataset_key") + protocol_entry=$(protocol-cache-ensure "$profile_json" "$outdir/../protocol/$protocol_key") + + ln -sf "$dataset_entry" "$outdir/dataset" + ln -sf "$protocol_entry" "$outdir/protocol" + + # -- genesis JSON files that don't change per run: assemble once ------------- + + # Byron and Shelley change per run (startTime/systemStart) so they are + # assembled in derive-from-cache-chunks instead. + + # Alonzo: protocol parameters only, no dataset fields. + cat "$protocol_entry/alonzo.protocol.json" > "$outdir/genesis.alonzo.json" + + # Conway: protocol parameters + dataset. + { + cat "$protocol_entry/conway.protocol.json" | sed 's/}$//' + printf ',' + tail -c +2 "$dataset_entry/conway.dataset.json" + } > "$outdir/genesis.conway.json" + + # Dijkstra: protocol parameters only, no dataset fields. + cat "$protocol_entry/dijkstra.protocol.json" > "$outdir/genesis.dijkstra.json" + + info genesis "genesis cache entry created in $outdir" +} + +# Populate the run directory from the genesis cache entry. +# Everything is symlinked except genesis.shelley.json which is assembled +# fresh per run (because of systemStart). Pure cat/sed/printf for shelley. +derive-from-cache-chunks() { + local profile_json=${1:?$usage} + local timing=${2:?$usage} + local cache_entry=${3:?$usage} + local outdir=${4:?$usage} # output directory (run dir's genesis/, e.g. run/current/genesis). + + mkdir -p "$outdir" + + local preset + preset=$(profile preset "$profile_json") + if [[ -n "$preset" ]]; then + progress "genesis" "instantiating from preset $(with_color white "$preset"): $cache_entry" + cp -f "$cache_entry"/genesis*.json "$outdir" + return + fi + + progress "genesis" "deriving from cache: $cache_entry -> $outdir" + + ln -s "$profile_json" "$outdir"/profile.json + ln -s "$cache_entry" "$outdir"/cache-entry + ln -s "$cache_entry"/cache.key "$outdir" + ln -s "$cache_entry"/cache.key.input "$outdir" + ln -s "$cache_entry"/layout.version "$outdir" + + # Key directories as symlinks from cache (create-testnet-data output names). + for keydir in byron-gen-command drep-keys pools-keys stake-delegators utxo-keys + do + ln -s "$cache_entry/dataset/$keydir" "$outdir/$keydir" + done + + # Genesis JSON files that don't change per run as symlinks from cache. + ln -s "$cache_entry"/genesis.alonzo.json "$outdir"/genesis.alonzo.json + ln -s "$cache_entry"/genesis.conway.json "$outdir"/genesis.conway.json + ln -s "$cache_entry"/genesis.dijkstra.json "$outdir"/genesis.dijkstra.json + + # Byron and Shelley change per run (startTime / systemStart). + # Assembled from protocol + dataset + per-run timing using cat/sed/printf. + local system_start_epoch system_start + system_start_epoch=$(jq -r '.start' <<<"$timing") + system_start=$(jq -r '.systemStart' <<<"$timing") + + # Byron (always small) + { + cat "$cache_entry/protocol/byron.protocol.json" | sed 's/}$//' + printf ',"startTime":%s' "$system_start_epoch" + printf ',' + tail -c +2 "$cache_entry/dataset/byron.dataset.json" + } > "$outdir/genesis.byron.json" + + # Shelley (can be 500MB+) + { + cat "$cache_entry/protocol/shelley.protocol.json" | sed 's/}$//' + printf ',"systemStart":"%s"' "$system_start" + printf ',' + tail -c +2 "$cache_entry/dataset/shelley.dataset.json" + } > "$outdir/genesis.shelley.json" +} + +# ============================================================================== +# Dataset cache (level 1) +# ============================================================================== + +# Build the dataset cache key from a profile JSON. +dataset-cache-key() { + local profile_json=$1 + + local params_key + params_key=$(jq -r ' + "pools-" + (.composition.n_pools | tostring) + + "-stake-delegators-" + (.derived.delegators_effective | tostring) + + "-drep-keys-" + (.genesis.dreps | tostring) + + "-stuffed-utxo-" + (.derived.utxo_stuffed | tostring) + + "-utxo-keys-" + (.genesis.utxo_keys | tostring) + + "-total-supply-" + (.derived.supply_total | tostring) + + "-delegated-supply-" + (.derived.supply_delegated | tostring) + + "-testnet-magic-" + (.genesis.network_magic | tostring) + ' "$profile_json") + + local hash + hash=$(echo "$params_key" | sha1sum | cut -c-7) + + echo "${params_key}-${hash}" +} + +# Ensure a dataset cache entry exists. If not, create it. +# Returns the cache entry path on stdout. +dataset-cache-ensure() { + local profile_json=$1 + local outdir=$2 + local zero_dir="$global_basedir/profile/presets/zero/genesis" + + if test -d "$outdir" + then + info genesis "dataset cache hit: $outdir" + else + info genesis "dataset cache miss: $outdir" + # Create in a temporary directory and move if no errors. + local tmpdir + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + # Extract dataset parameters from canonical profile fields + local era pools delegators dreps stuffed utxo_keys total_supply delegated_supply magic + era=$(jq --raw-output '.era' "$profile_json") + pools=$(jq --raw-output '.composition.n_pools' "$profile_json") + delegators=$(jq --raw-output '.derived.delegators_effective' "$profile_json") + dreps=$(jq --raw-output '.genesis.dreps' "$profile_json") + stuffed=$(jq --raw-output '.derived.utxo_stuffed' "$profile_json") + utxo_keys=$(jq --raw-output '.genesis.utxo_keys' "$profile_json") + total_supply=$(jq --raw-output '.derived.supply_total' "$profile_json") + delegated_supply=$(jq --raw-output '.derived.supply_delegated' "$profile_json") + magic=$(jq --raw-output '.genesis.network_magic' "$profile_json") + # Run `create-testnet-data` with zeroed specs (see profile/presets/zero). + progress genesis "creating dataset cache: $outdir" + cardano-cli "$era" genesis create-testnet-data \ + --spec-shelley "$zero_dir/shelley-genesis.json" \ + --spec-alonzo "$zero_dir/alonzo-genesis.json" \ + --spec-conway "$zero_dir/conway-genesis.json" \ + --out-dir "$tmpdir" \ + --start-time "1970-01-01T00:00:00Z" \ + --testnet-magic "$magic" \ + --total-supply "$total_supply" \ + --delegated-supply "$delegated_supply" \ + --utxo-keys "$utxo_keys" \ + --pools "$pools" \ + --stake-delegators "$delegators" \ + --drep-keys "$dreps" \ + --stuffed-utxo "$stuffed" + # Extract dataset fields as compact JSON (one jq pass per file) + jq -c '{bootStakeholders, heavyDelegation, nonAvvmBalances}' "$tmpdir/byron-genesis.json" \ + > "$tmpdir/byron.dataset.json" + jq -c '{initialFunds, staking, maxLovelaceSupply}' "$tmpdir/shelley-genesis.json" \ + > "$tmpdir/shelley.dataset.json" + jq -c '{initialDReps}' "$tmpdir/conway-genesis.json" \ + > "$tmpdir/conway.dataset.json" + # Ensure all key directories exist (even if `create-testnet-data` didn't create them) + for keydir in byron-gen-command drep-keys pools-keys stake-delegators utxo-keys + do + mkdir -p "$tmpdir/$keydir" + done + # TODO: remove the raw genesis files produced by create-testnet-data ??? + # Move the whole tmpdir to the cache entry + mkdir -p "$(dirname "$outdir")" + mv "$tmpdir" "$outdir" + trap - EXIT + info genesis "dataset cached: $outdir" + fi + + echo "$outdir" +} + +# ============================================================================== +# Protocol cache (level 2) +# ============================================================================== + +# Protocol cache key: slotLength, epochLength, activeSlotsCoeff (human-readable) +# followed by a hash of all protocol parameter fields. +protocol-cache-key() { + local profile_json=$1 + local slot epoch active hash + slot=$(jq -r '.genesis.shelley.slotLength' "$profile_json") + epoch=$(jq -r '.genesis.shelley.epochLength' "$profile_json") + active=$(jq -r '.genesis.shelley.activeSlotsCoeff' "$profile_json") + hash=$(jq -c -S '{ + byron: ( .genesis.byron // {} ), + shelley: ( .genesis.shelley ), + alonzo: ( .genesis.alonzo ), + conway: ( .genesis.conway // {} ), + dijkstra: ( .genesis.dijkstra // {} ) + }' "$profile_json" | sha1sum | cut -c-7) + echo "slot-${slot}-epoch-${epoch}-active-${active}-${hash}" +} + +# Ensure a protocol cache entry exists. If not, create it. +# Returns the cache entry path on stdout. +protocol-cache-ensure() { + local profile_json=$1 + local outdir=$2 + + if test -d "${outdir}" + then + info genesis "protocol parameters cache hit: ${outdir}" + else + info genesis "protocol parameters cache miss: ${outdir}" + mkdir -p "${outdir}" + # Generate specs from profile and strip dataset fields + systemStart. + # All specs are small (<10KB), using `jq` is fine here. + # MUST be compact (-c) because the merge step uses sed 's/}$//' which + # only works on single-line JSON. + genesis spec byron "${profile_json}" \ + | jq -c 'del(.bootStakeholders, .heavyDelegation, .nonAvvmBalances, .startTime)' \ + > "${outdir}/byron.protocol.json" + genesis spec shelley "${profile_json}" \ + | jq -c 'del(.initialFunds, .staking, .maxLovelaceSupply, .systemStart)' \ + > "${outdir}/shelley.protocol.json" + genesis spec alonzo "${profile_json}" \ + | jq -c '.' \ + > "${outdir}/alonzo.protocol.json" + genesis spec conway "${profile_json}" \ + | jq -c 'del(.initialDReps)' \ + > "${outdir}/conway.protocol.json" + genesis spec dijkstra "${profile_json}" \ + | jq -c '.' \ + > "${outdir}/dijkstra.protocol.json" + info genesis "protocol parameters cached: ${outdir}" + fi + + echo "${outdir}" +} + diff --git a/nix/workbench/genesis/genesis-jq.sh b/nix/workbench/genesis/genesis-jq.sh new file mode 100644 index 00000000000..0a09903ca58 --- /dev/null +++ b/nix/workbench/genesis/genesis-jq.sh @@ -0,0 +1,408 @@ +# shellcheck shell=bash +# +# Legacy (jq-based) genesis backend. +# Used when WB_MODULAR_GENESIS=0 and WB_DATASET_GENESIS=0 (the default). +# +# Implements the backend interface: +# profile-cache-key-input-jq, profile-cache-key-jq, +# spec-jq, pool-relays-jq, +# genesis-create-jq, derive-from-cache-jq + +profile-cache-key-input-jq() { + set -euo pipefail + local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} + + local args=( + --slurpfile profile "$profile_json" + --arg profile_json "$profile_json" + --sort-keys + --null-input + -L "$global_basedir/profile" + -L "$global_basedir/genesis" + ) + jq 'include "genesis"; profile_genesis_cache_key($profile[0]; $profile_json)' "${args[@]}" +} + +profile-cache-key-jq() { + set -euo pipefail + local usage="USAGE: wb genesis profile-cache-key PROFILE-JSON" + local profile_json=${1:?$usage} + local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} + + local args=( + --arg params_hash "$(genesis profile-cache-key-input "$profile_json" | sha1sum | cut -c-7)" + --slurpfile profile "$profile_json" + --raw-output + -L "$global_basedir/profile" + -L "$global_basedir/genesis" + ) + jq 'include "genesis"; profile_genesis_cache_entry_name($profile[0]; $params_hash)' "${args[@]}" "$profile_json" +} + +spec-jq() { + set -euo pipefail + local era=${1:?missing era} + local profile_json=${2:?missing profile_json} + # not needed in the jq version + # local node_specs=${3:?missing node_specs} + + case "$1" in + byron) + # Byron base matches what master's genesis-byron-jq passed to + # `cardano-cli byron genesis genesis` as protocol parameters, + # plus the --k and --protocol-magic from the profile. + jq --null-input --slurpfile prof "$profile_json" ' + $prof[0] as $p | { + avvmDistr: {}, + blockVersionData: { + heavyDelThd: "300000", + maxBlockSize: "641000", + maxHeaderSize: "200000", + maxProposalSize: "700", + maxTxSize: "4096", + mpcThd: "200000", + scriptVersion: 0, + slotDuration: "20000", + softforkRule: { + initThd: "900000", + minThd: "600000", + thdDecrement: "100000" + }, + txFeePolicy: { + multiplier: "439460", + summand: "0" + }, + unlockStakeEpoch: "184467", + updateImplicit: "10000", + updateProposalThd: "100000", + updateVoteThd: "100000" + }, + protocolConsts: { + k: $p.genesis.parameter_k, + protocolMagic: $p.genesis.network_magic + }, + startTime: 0 + } + ' + ;; + shelley) + # We pass the network magic as an argument to cardano-cli, but, despite being required, + # it is only used to amend a default template while we are bringing our own. So we put it into + # the template too. + jq '$prof[0] as $p + | . * ($p.genesis.shelley // {}) + | .networkMagic |= $p.genesis.network_magic + ' --slurpfile prof "$profile_json" \ + "$global_basedir"/profile/presets/mainnet/genesis/genesis-shelley.json + ;; + alonzo) + jq '$prof[0] as $p | . * ($p.genesis.alonzo // {})' \ + --slurpfile prof "$profile_json" \ + "$global_basedir"/profile/presets/mainnet/genesis/genesis.alonzo.json + ;; + conway) + jq '$prof[0] as $p | . * ($p.genesis.conway // {})' \ + --slurpfile prof "$profile_json" \ + "$global_basedir"/profile/presets/mainnet/genesis/genesis.conway.json + ;; + dijkstra) + jq '$prof[0] as $p | . * ($p.genesis.dijkstra // {})' \ + --slurpfile prof "$profile_json" \ + "$global_basedir"/profile/presets/zero/genesis/dijkstra-genesis.json + ;; + *) + echo unknown era + exit 1 + ;; + esac +} + +pool-relays-jq() { + set -euo pipefail + local profile_json=${1:?missing profile_json} + # not needed in the jq version + # local node_specs=${2:?missing node_specs} + + jq ' to_entries + | map( select(.value.kind == "pool") ) + | map + ({ key: (.value.i | tostring) + , value: + [{ "single host name": + { dnsName: .key + , port: .value.port + } + } + ] + }) + | from_entries + ' "$node_specs" +} + +# Entry point for genesis creation. +genesis-create-jq() { + if [[ $WB_CREATE_TESTNET_DATA -eq 1 ]]; then + genesis-create-testnet-data "$@" + else + genesis-create-staked "$@" + fi +} + +genesis-create-staked() { + local profile_json=$1 + local dir=$2 + + rm -rf "$dir"/{*-keys,byron,pools,nodes,*.json,*.params,*.version} + mkdir -p "$dir" + + genesis spec shelley "$profile_json" >"$dir/genesis.spec.json" + genesis spec alonzo "$profile_json" >"$dir/genesis.alonzo.spec.json" + genesis spec conway "$profile_json" >"$dir/genesis.conway.spec.json" + genesis pool-relays "$profile_json" >"$dir/pool-relays.json" + + read -r -a args <<<"$(jq --raw-output '.cli_args.createStakedArgs | join(" ")' "$profile_json")" + create_staked_args=( + --genesis-dir "$dir" + --relay-specification-file "$dir/pool-relays.json" + "${args[@]}" + ) + progress "genesis" "$(colorise cardano-cli genesis create-staked "${create_staked_args[@]}")" + cardano-cli genesis create-staked "${create_staked_args[@]}" + + # Normalize create-staked output to match create-testnet-data layout, + # so the rest of the workbench uses a single format. + normalize-create-staked-output "$dir" +} + +# Rearrange create-staked output to match create-testnet-data layout. +# +# create-staked produces: +# pools/cold1.skey, pools/kes1.skey, ... (flat, 1-indexed) +# delegate-keys/delegate1.skey, ... (BFT keys) +# stake-delegator-keys/ (different name) +# genesis.json (different name) +# +# create-testnet-data produces: +# pools-keys/pool1/cold.skey, ... (subdirs, 1-indexed) +# stake-delegators/delegator1/ (different name) +# shelley-genesis.json (different name) +# drep-keys/ byron-gen-command/ (not in create-staked) +# +normalize-create-staked-output() { + local dir=$1 + + # Genesis JSON naming + mv "$dir"/genesis.json "$dir"/genesis.shelley.json + mv "$dir"/genesis.spec.json "$dir"/genesis.shelley.spec.json + + # Pool keys: pools/cold1.skey → pools-keys/pool1/cold.skey + if [[ -d "$dir"/pools ]]; then + mkdir -p "$dir"/pools-keys + # Find the highest pool index + local max_pool=0 + for f in "$dir"/pools/cold*.skey; do + [[ -f "$f" ]] || continue + local n=${f##*cold}; n=${n%.skey} + (( n > max_pool )) && max_pool=$n + done + for n in $(seq 1 "$max_pool"); do + local pooldir="$dir/pools-keys/pool${n}" + mkdir -p "$pooldir" + # Move each key type, matching create-testnet-data naming + [[ -f "$dir/pools/cold${n}.skey" ]] && mv "$dir/pools/cold${n}.skey" "$pooldir/cold.skey" + [[ -f "$dir/pools/cold${n}.vkey" ]] && mv "$dir/pools/cold${n}.vkey" "$pooldir/cold.vkey" + [[ -f "$dir/pools/kes${n}.skey" ]] && mv "$dir/pools/kes${n}.skey" "$pooldir/kes.skey" + [[ -f "$dir/pools/kes${n}.vkey" ]] && mv "$dir/pools/kes${n}.vkey" "$pooldir/kes.vkey" + [[ -f "$dir/pools/vrf${n}.skey" ]] && mv "$dir/pools/vrf${n}.skey" "$pooldir/vrf.skey" + [[ -f "$dir/pools/vrf${n}.vkey" ]] && mv "$dir/pools/vrf${n}.vkey" "$pooldir/vrf.vkey" + [[ -f "$dir/pools/opcert${n}.cert" ]] && mv "$dir/pools/opcert${n}.cert" "$pooldir/opcert.cert" + [[ -f "$dir/pools/opcert${n}.counter" ]] && mv "$dir/pools/opcert${n}.counter" "$pooldir/opcert.counter" + done + rm -rf "$dir"/pools + fi + + # Stake delegator keys: stake-delegator-keys/ → stake-delegators/ + if [[ -d "$dir"/stake-delegator-keys ]]; then + mv "$dir"/stake-delegator-keys "$dir"/stake-delegators + fi + + # create-staked does not produce a byron genesis; generate from spec. + genesis spec byron "$profile_json" > "$dir"/genesis.byron.json + + # Ensure directories that create-testnet-data always creates + mkdir -p "$dir"/{byron-gen-command,drep-keys,pools-keys,stake-delegators,utxo-keys} + + info genesis "normalized create-staked output to create-testnet-data layout" +} + +genesis-create-testnet-data() { + local profile_json=$1 + local dir=$2 + + mkdir -p "$dir" + + genesis spec shelley "$profile_json" >"$dir/shelley-genesis.spec.json" + genesis spec alonzo "$profile_json" >"$dir/alonzo-genesis.spec.json" + genesis spec conway "$profile_json" >"$dir/conway-genesis.spec.json" + genesis pool-relays "$profile_json" >"$dir/pool-relays.json" + + local era + era=$(jq --raw-output '.era' "$profile_json") + + # TODO if profile_json.composition.dense_pool_density != 1 -> create-testnet-data does not support dense pools + read -r -a args <<<"$(jq --raw-output '.cli_args.createTestnetDataArgs | join(" ")' "$profile_json")" + create_testnet_data_args=( + --spec-shelley "$dir/shelley-genesis.spec.json" + --spec-alonzo "$dir/alonzo-genesis.spec.json" + --spec-conway "$dir/conway-genesis.spec.json" + --relays "$dir/pool-relays.json" + --out-dir "$dir" + "${args[@]}" + ) + progress genesis "$(colorise cardano-cli "$era" create-testnet-data "${create_testnet_data_args[@]}")" + cardano-cli "$era" genesis create-testnet-data "${create_testnet_data_args[@]}" + + # Normalize create-testnet-data output names to genesis..json + mv "$dir"/byron-genesis.json "$dir"/genesis.byron.json + mv "$dir"/shelley-genesis.json "$dir"/genesis.shelley.json + mv "$dir"/alonzo-genesis.json "$dir"/genesis.alonzo.json + mv "$dir"/conway-genesis.json "$dir"/genesis.conway.json + mv "$dir"/dijkstra-genesis.json "$dir"/genesis.dijkstra.json + + local is_voting + is_voting=$(jq --raw-output '.workloads | any( .name == "voting")' "$profile_json") + if [[ "$is_voting" == "true" ]]; + then + info genesis "voting workload specified - keeping one stake key per producer" + mv "$dir/stake-delegators" "$dir/stake-delegators.bak" + mkdir "$dir/stake-delegators" + local pools + pools="$(jq --raw-output '.composition.n_pools' "${profile_json}")" + for i in $(seq 1 "$pools") + do + if test -d "$dir/stake-delegators.bak/delegator${i}" + then + local from_dir to_dir + from_dir="$dir/stake-delegators.bak/delegator${i}" + to_dir="$dir/stake-delegators/delegator$((i - 1))" + mkdir "$to_dir" + cp "$from_dir"/{payment,staking}.{skey,vkey} "$to_dir"/ + fi + done + rm "$dir/stake-delegators.bak" -rf + info genesis "voting workload specified - skipping deletion of DRep keys" + else + info genesis "removing delegator keys." + rm "$dir/stake-delegators" -rf + info genesis "removing dreps keys." + rm "$dir"/drep-keys -rf + fi + + info genesis "create-testnet-data genesis available in $dir" +} + +derive-from-cache-jq() { + local usage="USAGE: wb genesis derive-from-cache PROFILE-OUT TIMING-JSON-EXPR CACHE-ENTRY-DIR OUTDIR" + local profile=${1:?$usage} + local timing=${2:?$usage} + local cache_entry=${3:?$usage} + local outdir=${4:?$usage} # output directory (run dir's genesis/, e.g. run/current/genesis). + + mkdir -p "$outdir" + + local preset + preset=$(profile preset "$profile") + if [[ -n "$preset" ]]; then + progress "genesis" "instantiating from preset $(with_color white "$preset"): $cache_entry" + cp -f "$cache_entry"/genesis*.json "$outdir" + return + fi + + progress "genesis" "deriving from cache: $cache_entry -> $outdir" + + ln -s "$profile" "$outdir"/profile.json + ln -s "$cache_entry" "$outdir"/cache-entry + ln -s "$cache_entry"/cache.key "$outdir" + ln -s "$cache_entry"/cache.key.input "$outdir" + ln -s "$cache_entry"/layout.version "$outdir" + + # Key directories as symlinks using create-testnet-data output names. + for keydir in byron-gen-command drep-keys pools-keys stake-delegators utxo-keys + do + if [[ -d "$cache_entry/$keydir" ]]; then + ln -s "$cache_entry/$keydir" "$outdir/$keydir" + else + mkdir -p "$outdir/$keydir" + fi + done + + ## genesis + cp "$cache_entry"/genesis*.json "$outdir" + chmod u+w "$outdir"/genesis*.json + + local dir="$outdir" + + ## -- finalise: patch genesis files with per-run values --------------------- + + local profile_json=$profile + local dir=$outdir + + progress "genesis" "finalizing retrieved cache entry in: $outdir" + + local system_start_epoch + system_start_epoch="$(jq '.start' -r <<<"$timing")" + + # Byron: patch startTime (same approach as shelley's systemStart below) + jq --argjson start "$system_start_epoch" '.startTime = $start' \ + "$dir"/genesis.byron.json | + sponge "$dir"/genesis.byron.json + + # The genesis cache entry in $outdir needs post-processing: + # * the system start date might get an adjustment offset into the future + # * some workbench profile content not captured by the genesis cache key will be patched in + + # For devs: this should cover all fields modified by + # * nix/workbench/profile/pparams/delta-*.jq (workbench) + # * bench/cardano-profile/data/genesis/overlays/*.json (cardano-profile) + # These modifications are small deltas to base profiles, which do not, and should + # not, result in recreation of the entire staked genesis, as that is large on-disk + # and takes long to create. + + # Shelley: startTime, protocolVersion, maxBlockBodySize + jq '$prof[0].genesis.shelley as $shey + | $shey.protocolParams.protocolVersion as $pver + | $shey.protocolParams.maxBlockBodySize as $bsize + | . * { systemStart: $timing.systemStart } + | if $pver != null then . * { protocolParams: { protocolVersion: $pver } } else . end + | if $bsize != null then . * { protocolParams: { maxBlockBodySize: $bsize } } else . end' \ + --argjson timing "$timing" \ + --slurpfile prof "$profile_json" \ + "$dir"/genesis.shelley.json | + sponge "$dir"/genesis.shelley.json + + # Alonzo: Execution budgets + # NB. PlutusV1 and PlutusV2 cost models are *NOT* covered here; they're encoded + # as key-value-map in the profile, but need to be [Integer] in genesis. + jq '$prof[0].genesis.alonzo as $alzo + | $alzo.maxBlockExUnits.exUnitsMem as $bl_mem + | $alzo.maxBlockExUnits.exUnitsSteps as $bl_steps + | $alzo.maxTxExUnits.exUnitsMem as $tx_mem + | $alzo.maxTxExUnits.exUnitsSteps as $tx_steps + | if $bl_mem != null then . * { maxBlockExUnits: { memory: $bl_mem} } else . end + | if $bl_steps != null then . * { maxBlockExUnits: { steps: $bl_steps} } else . end + | if $tx_mem != null then . * { maxTxExUnits: { memory: $tx_mem} } else . end + | if $tx_steps != null then . * { maxTxExUnits: { steps: $tx_steps} } else . end' \ + --slurpfile prof "$profile_json" \ + "$dir"/genesis.alonzo.json | + sponge "$dir"/genesis.alonzo.json + + # Conway: plutusV3CostModel + jq '$prof[0].genesis.conway as $coay + | $coay.plutusV3CostModel as $pv3cost + | if $pv3cost != null then . * { plutusV3CostModel: $pv3cost } else . end' \ + --slurpfile prof "$profile_json" \ + "$dir"/genesis.conway.json | + sponge "$dir"/genesis.conway.json + +} + diff --git a/nix/workbench/genesis/genesis-modular.sh b/nix/workbench/genesis/genesis-modular.sh new file mode 100644 index 00000000000..2a7ff7249d7 --- /dev/null +++ b/nix/workbench/genesis/genesis-modular.sh @@ -0,0 +1,79 @@ +# shellcheck shell=bash +# +# Modular (Nix-based) genesis backend. +# Used when WB_MODULAR_GENESIS=1. +# +# Implements the backend interface: +# profile-cache-key-input-modular, profile-cache-key-modular, +# spec-modular, pool-relays-modular, +# genesis-create-modular, derive-from-cache-modular + +profile-cache-key-input-modular() { + set -euo pipefail + local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} + + # nix wants absolute paths + profile_json=$(realpath "$profile_json") + + # NOTE: jq is only used for formatting + evaluate --profile "${profile_json}" genesis.cache-key-input | jq +} + +profile-cache-key-modular() { + local usage="USAGE: wb genesis profile-cache-key PROFILE-JSON" + local profile_json=${1:?$usage} + + # nix wants absolute paths + profile_json=$(realpath "$profile_json") + + # NOTE: + # - the hash is different because nix cannot reproduce jq's pretty-printing + # - jq is only used for formatting + evaluate --profile "${profile_json}" genesis.cache-key | jq -r +} + +spec-modular() { + set -euo pipefail + local era=${1:?missing era} + local profile_json=${2:?missing profile_json} + local node_specs=${3:?missing node_specs} + + # dijkstra is not in the Nix modules, fall back to jq + if [[ "$era" == "dijkstra" ]]; then + spec-jq "$@" + return + fi + + # nix wants absolute paths + profile_json=$(realpath "$profile_json") + node_specs=$(realpath "$node_specs") + + # TODO: check for errors + # NOTE: jq is only used for to keep the same formatting as the jq version + evaluate --profile "$profile_json" --node-specs "$node_specs" "genesis.$era" | jq +} + +pool-relays-modular() { + set -euo pipefail + local profile_json=${1:?missing profile_json} + local node_specs=${2:?missing node_specs} + + # nix wants absolute paths + profile_json=$(realpath "$profile_json") + node_specs=$(realpath "$node_specs") + + # NOTE: jq is only used for to keep the same formatting as the jq version + evaluate --profile "$profile_json" --node-specs "$node_specs" genesis.pool-relays | jq +} + +# Entry point for genesis creation (delegates to legacy; modular only +# replaces spec/cache-key/byron generation, not the create-testnet-data call). +genesis-create-modular() { + genesis-create-jq "$@"; +} + +# Same derive-from-cache as legacy. +derive-from-cache-modular() { + derive-from-cache-jq "$@"; +} + diff --git a/nix/workbench/genesis/genesis.sh b/nix/workbench/genesis/genesis.sh index 580f08c8754..501dc48cb4e 100644 --- a/nix/workbench/genesis/genesis.sh +++ b/nix/workbench/genesis/genesis.sh @@ -3,6 +3,14 @@ global_genesis_format_version=October-13-2025 +# Resolve genesis backend once (at source time, no output). +# Each backend file defines: spec-*, pool-relays-*, profile-cache-key-*, +# profile-cache-key-input-*, genesis-create-*, derive-from-cache-* +if [[ ${WB_GENESIS_CHUNKS:-0} -eq 1 ]]; then genesis_backend=chunks +elif [[ ${WB_MODULAR_GENESIS:-0} -eq 1 ]]; then genesis_backend=modular +else genesis_backend=jq +fi + usage_genesis() { usage "genesis" "Genesis" < "$dir"/cache.key.input @@ -197,117 +168,48 @@ case "$op" in cat <<<"$global_genesis_format_version" > "$dir"/layout.version ;; - derive-from-cache ) - local usage="USAGE: wb genesis $op PROFILE-OUT TIMING-JSON-EXPR CACHE-ENTRY-DIR OUTDIR" - local profile=${1:?$usage} - local timing=${2:?$usage} - local cache_entry=${3:?$usage} - local outdir=${4:?$usage} - - mkdir -p "$outdir" - - local preset - preset=$(profile preset "$profile") - if [[ -n "$preset" ]]; then - progress "genesis" "instantiating from preset $(with_color white "$preset"): $cache_entry" - mkdir -p "$outdir"/byron - cp -f "$cache_entry"/genesis*.json "$outdir" - cp -f "$cache_entry"/byron/*.json "$outdir"/byron - return - fi - - progress "genesis" "deriving from cache: $cache_entry -> $outdir" - - ln -s "$profile" "$outdir"/profile.json - ln -s "$cache_entry" "$outdir"/cache-entry - ln -s "$cache_entry"/cache.key "$outdir" - ln -s "$cache_entry"/cache.key.input "$outdir" - ln -s "$cache_entry"/layout.version "$outdir" - - # create-testnet-data does not create these directories if there are no keys in them - [[ -d "$dir"/delegate-keys ]] && ln -s "$cache_entry"/delegate-keys "$outdir" - [[ -d "$dir"/genesis-keys ]] && ln -s "$cache_entry"/genesis-keys "$outdir" - - cp -a "$cache_entry"/node-keys "$outdir" - chmod -R go-rwx "$outdir"/node-keys + # -- Dispatched to backend ------------------------------------------------- - ln -s "$cache_entry"/pools "$outdir" - ln -s "$cache_entry"/stake-delegator-keys "$outdir" - ln -s "$cache_entry"/utxo-keys "$outdir" - - ## genesis - cp "$cache_entry"/genesis*.json "$outdir" - chmod u+w "$outdir"/genesis*.json - - genesis finalise-cache-entry "$profile" "$timing" "$outdir" + # Called by: genesis.sh prepare-cache-entry. + # $1: profile JSON file path (e.g. /nix/store/.../profile.json). + # Returns: JSON or string written to cache.key.input for debugging. + profile-cache-key-input ) + "profile-cache-key-input-$genesis_backend" "$@" ;; - finalise-cache-entry ) - local usage="USAGE: wb genesis $op PROFILE-JSON TIMING-JSON-EXPR DIR" - local profile_json=${1:?$usage} - local timing=${2:?$usage} - local dir=${3:?$usage} + # Called by: genesis.sh prepare-cache-entry. + # $1: profile JSON file path (e.g. /nix/store/.../profile.json). + # Returns: short cache directory name (e.g. "ci-test-coay-1c5c3ba-8444286"). + profile-cache-key ) + "profile-cache-key-$genesis_backend" "$@" + ;; - if profile has-preset "$profile_json"; then - return - fi + # Called by: genesis-jq.sh (genesis-create-staked, genesis-create-testnet-data), + # genesis-dataset.sh (protocol-cache-ensure). + # $1: era name (byron, shelley, alonzo, conway, dijkstra). + # $2: profile JSON file path. + # Returns: JSON spec on stdout (mainnet preset merged with profile overrides). + spec ) + "spec-$genesis_backend" "$@" + ;; - progress "genesis" "finalizing retrieved cache entry in: $outdir" - - local system_start_epoch - system_start_epoch="$(jq '.start' -r <<<"$timing")" - - genesis-byron "$system_start_epoch" "$dir" "$profile_json" - - # The genesis cache entry in $outdir needs post-processing: - # * the system start date might get an adjustment offset into the future - # * some workbench profile content not captured by the genesis cache key will be patched in - - # For devs: this should cover all fields modified by - # * nix/workbench/profile/pparams/delta-*.jq (workbench) - # * bench/cardano-profile/data/genesis/overlays/*.json (cardano-profile) - # These modifications are small deltas to base profiles, which do not, and should - # not, result in recreation of the entire staked genesis, as that is large on-disk - # and takes long to create. - - # Shelley: startTime, protocolVersion, maxBlockBodySize - jq '$prof[0].genesis.shelley as $shey - | $shey.protocolParams.protocolVersion as $pver - | $shey.protocolParams.maxBlockBodySize as $bsize - | . * { systemStart: $timing.systemStart } - | if $pver != null then . * { protocolParams: { protocolVersion: $pver } } else . end - | if $bsize != null then . * { protocolParams: { maxBlockBodySize: $bsize } } else . end' \ - --argjson timing "$timing" \ - --slurpfile prof "$profile_json" \ - "$dir"/genesis-shelley.json | - sponge "$dir"/genesis-shelley.json - - # Alonzo: Execution budgets - # NB. PlutusV1 and PlutusV2 cost models are *NOT* covered here; they're encoded - # as key-value-map in the profile, but need to be [Integer] in genesis. - jq '$prof[0].genesis.alonzo as $alzo - | $alzo.maxBlockExUnits.exUnitsMem as $bl_mem - | $alzo.maxBlockExUnits.exUnitsSteps as $bl_steps - | $alzo.maxTxExUnits.exUnitsMem as $tx_mem - | $alzo.maxTxExUnits.exUnitsSteps as $tx_steps - | if $bl_mem != null then . * { maxBlockExUnits: { memory: $bl_mem} } else . end - | if $bl_steps != null then . * { maxBlockExUnits: { steps: $bl_steps} } else . end - | if $tx_mem != null then . * { maxTxExUnits: { memory: $tx_mem} } else . end - | if $tx_steps != null then . * { maxTxExUnits: { steps: $tx_steps} } else . end' \ - --slurpfile prof "$profile_json" \ - "$dir"/genesis.alonzo.json | - sponge "$dir"/genesis.alonzo.json - - # Conway: plutusV3CostModel - jq '$prof[0].genesis.conway as $coay - | $coay.plutusV3CostModel as $pv3cost - | if $pv3cost != null then . * { plutusV3CostModel: $pv3cost } else . end' \ - --slurpfile prof "$profile_json" \ - "$dir"/genesis.conway.json | - sponge "$dir"/genesis.conway.json + # Called by: genesis-jq.sh (genesis-create-staked, genesis-create-testnet-data). + # $1: profile JSON file path. + # Returns: JSON pool relays on stdout. + pool-relays ) + "pool-relays-$genesis_backend" "$@" + ;; + # Called by: run.sh. + # $1: profile JSON file path (run dir's profile.json, e.g. run/current/profile.json). + # $2: timing JSON string (from `profile allocate-time`). + # $3: genesis cache entry path (e.g. ~/.cache/cardano-workbench/genesis/ci-test-coay-...). + # $4: output directory (run dir's genesis/, e.g. run/current/genesis). + derive-from-cache ) + "derive-from-cache-$genesis_backend" "$@" ;; + * ) usage_genesis ;; @@ -315,459 +217,3 @@ case "$op" in esac } -profile-cache-key-input-jq() { - set -euo pipefail - local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} - - local args=( - --slurpfile profile "$profile_json" - --arg profile_json "$profile_json" - --sort-keys - --null-input - -L "$global_basedir/profile" - -L "$global_basedir/genesis" - ) - jq 'include "genesis"; profile_genesis_cache_key($profile[0]; $profile_json)' "${args[@]}" -} - -profile-cache-key-input-modular() { - set -euo pipefail - local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} - - # nix wants absolute paths - profile_json=$(realpath "$profile_json") - - # NOTE: jq is only used for formatting - evaluate --profile "${profile_json}" genesis.cache-key-input | jq -} - -profile-cache-key-jq() { - set -euo pipefail - local usage="USAGE: wb genesis profile-cache-key PROFILE-JSON" - local profile_json=${1:?$usage} - local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json} - - local args=( - --arg params_hash "$(genesis profile-cache-key-input "$profile_json" | sha1sum | cut -c-7)" - --slurpfile profile "$profile_json" - --raw-output - -L "$global_basedir/profile" - -L "$global_basedir/genesis" - ) - jq 'include "genesis"; profile_genesis_cache_entry_name($profile[0]; $params_hash)' "${args[@]}" "$profile_json" -} - -profile-cache-key-modular() { - local usage="USAGE: wb genesis profile-cache-key PROFILE-JSON" - local profile_json=${1:?$usage} - - # nix wants absolute paths - profile_json=$(realpath "$profile_json") - - # NOTE: - # - the hash is different because nix cannot reproduce jq's pretty-printing - # - jq is only used for formatting - evaluate --profile "${profile_json}" genesis.cache-key | jq -r -} - -spec-jq() { - set -euo pipefail - local era=${1:?missing era} - local profile_json=${2:?missing profile_json} - # not needed in the jq version - # local node_specs=${3:?missing node_specs} - - case "$1" in - shelley) - # We pass the network magic as an argument to cardano-cli, but, despite being required, - # it is only used to amend a default template while we are bringing our own. So we put it into - # the template too. - jq '$prof[0] as $p - | . * ($p.genesis.shelley // {}) - | .networkMagic |= $p.genesis.network_magic - ' --slurpfile prof "$profile_json" \ - "$global_basedir"/profile/presets/mainnet/genesis/genesis-shelley.json - ;; - alonzo) - jq '$prof[0] as $p | . * ($p.genesis.alonzo // {})' \ - --slurpfile prof "$profile_json" \ - "$global_basedir"/profile/presets/mainnet/genesis/genesis.alonzo.json - ;; - conway) - jq '$prof[0] as $p | . * ($p.genesis.conway // {})' \ - --slurpfile prof "$profile_json" \ - "$global_basedir"/profile/presets/mainnet/genesis/genesis.conway.json - ;; - *) - echo unknown era - exit 1 - ;; - esac -} - -spec-modular() { - set -euo pipefail - local era=${1:?missing era} - local profile_json=${2:?missing profile_json} - local node_specs=${3:?missing node_specs} - - # nix wants absolute paths - profile_json=$(realpath "$profile_json") - node_specs=$(realpath "$node_specs") - - # TODO: check for errors - # NOTE: jq is only used for to keep the same formatting as the jq version - evaluate --profile "$profile_json" --node-specs "$node_specs" "genesis.$era" | jq -} - -pool-relays-jq() { - set -euo pipefail - local profile_json=${1:?missing profile_json} - # not needed in the jq version - # local node_specs=${2:?missing node_specs} - - jq ' to_entries - | map( select(.value.kind == "pool") ) - | map - ({ key: (.value.i | tostring) - , value: - [{ "single host name": - { dnsName: .key - , port: .value.port - } - } - ] - }) - | from_entries - ' "$node_specs" -} - -pool-relays-modular() { - set -euo pipefail - local profile_json=${1:?missing profile_json} - local node_specs=${2:?missing node_specs} - - # nix wants absolute paths - profile_json=$(realpath "$profile_json") - node_specs=$(realpath "$node_specs") - - # NOTE: jq is only used for to keep the same formatting as the jq version - evaluate --profile "$profile_json" --node-specs "$node_specs" genesis.pool-relays | jq -} - -__KEY_ROOT= -Massage_the_key_file_layout_to_match_AWS() { - local profile_json=${1:?$usage} - local node_specs=${2:?$usage} - local dir=${3:?$usage} - local ids - local pool_density_map - - pool_density_map=$(topology density-map "$node_specs") - if [[ -z "$pool_density_map" ]]; then - fatal "failed: topology density-map '$node_specs'" - fi - info genesis "pool density map: $pool_density_map" - - __KEY_ROOT=$dir - - read -r -a ids <<< "$(jq 'keys | join(" ")' -cr <<< "$pool_density_map")" - - local bid=1 pid=1 did=1 ## (B)FT, (P)ool, (D)ense pool - for id in "${ids[@]}"; do - mkdir -p "$dir"/node-keys/cold - - #### cold keys (do not copy to production system) - if jqtest ".composition.dense_pool_density > 1" "$profile_json" && - jqtest ".[\"$id\"] > 1" <<<"$pool_density_map" - then ## Dense/bulk pool - info genesis "bulk pool $did -> node-$id" - cp -f "$(key_genesis bulk bulk "$did")" "$(key_depl bulk bulk "$id")" - did=$((did + 1)) - elif jqtest ".[\"$id\"] != 0" <<<"$pool_density_map" - then ## Singular pool - info genesis "pool $pid -> node-$id" - cp -f "$(key_genesis cold sig "$pid")" "$(key_depl cold sig "$id")" - cp -f "$(key_genesis cold ver "$pid")" "$(key_depl cold ver "$id")" - cp -f "$(key_genesis opcert cert "$pid")" "$(key_depl opcert none "$id")" - cp -f "$(key_genesis opcert count "$pid")" "$(key_depl cold count "$id")" - cp -f "$(key_genesis KES sig "$pid")" "$(key_depl KES sig "$id")" - cp -f "$(key_genesis KES ver "$pid")" "$(key_depl KES ver "$id")" - cp -f "$(key_genesis VRF sig "$pid")" "$(key_depl VRF sig "$id")" - cp -f "$(key_genesis VRF ver "$pid")" "$(key_depl VRF ver "$id")" - pid=$((pid + 1)) - else ## BFT node - info genesis "BFT $bid -> node-$id" - cp -f "$(key_genesis deleg sig "$bid")" "$(key_depl cold sig "$id")" - cp -f "$(key_genesis deleg ver "$bid")" "$(key_depl cold ver "$id")" - cp -f "$(key_genesis delegCert cert "$bid")" "$(key_depl opcert none "$id")" - cp -f "$(key_genesis deleg count "$bid")" "$(key_depl cold count "$id")" - cp -f "$(key_genesis delegKES sig "$bid")" "$(key_depl KES sig "$id")" - cp -f "$(key_genesis delegKES ver "$bid")" "$(key_depl KES ver "$id")" - cp -f "$(key_genesis delegVRF sig "$bid")" "$(key_depl VRF sig "$id")" - cp -f "$(key_genesis delegVRF ver "$bid")" "$(key_depl VRF ver "$id")" - bid=$((bid + 1)) - fi - done -} - -key_depl() { - local type=$1 kind=$2 id=$3 - case "$kind" in - bulk ) suffix='.creds';; - cert ) suffix='.cert';; - count ) suffix='.counter';; - none ) suffix=;; - sig ) suffix='.skey';; - ver ) suffix='.vkey';; - * ) fatal "key_depl: unknown key kind: '$kind'";; esac - case "$type" in - bulk ) stem=node-keys/bulk$id;; - cold ) stem=node-keys/cold/operator$id;; - opcert ) stem=node-keys/node$id.opcert;; - KES ) stem=node-keys/node-kes$id;; - VRF ) stem=node-keys/node-vrf$id;; - * ) fatal "key_depl: unknown key type: '$type'";; esac - echo "$__KEY_ROOT/$stem$suffix" -} - -key_genesis() { - local type=$1 kind=$2 id=$3 - case "$kind" in - bulk ) suffix='.creds';; - cert ) suffix='.cert';; - count ) suffix='.counter';; - none ) suffix=;; - sig ) suffix='.skey';; - ver ) suffix='.vkey';; - * ) fatal "key_genesis: unknown key kind: '$kind'";; esac - case "$type" in - bulk ) stem=pools/bulk$id;; - cold ) stem=pools/cold$id;; - opcert ) stem=pools/opcert$id;; - KES ) stem=pools/kes$id;; - VRF ) stem=pools/vrf$id;; - deleg ) stem=delegate-keys/delegate$id;; - delegCert )stem=delegate-keys/opcert$id;; - delegKES ) stem=delegate-keys/delegate$id.kes;; - delegVRF ) stem=delegate-keys/delegate$id.vrf;; - * ) fatal "key_genesis: unknown key type: '$type'";; esac - echo "$__KEY_ROOT/$stem$suffix" -} - -genesis-byron() { - set -euo pipefail - if [[ $WB_MODULAR_GENESIS -eq 1 ]]; then - genesis-byron-modular "$@" - else - genesis-byron-jq "$@" - fi -} - -genesis-byron-jq() { - local system_start_epoch=$1 - local dir=$2 - local profile_json=$3 - - jq ' - { heavyDelThd: "300000" - , maxBlockSize: "641000" - , maxHeaderSize: "200000" - , maxProposalSize: "700" - , maxTxSize: "4096" - , mpcThd: "200000" - , scriptVersion: 0 - , slotDuration: "20000" - , softforkRule: - { initThd: "900000" - , minThd: "600000" - , thdDecrement: "100000" - } - , txFeePolicy: - { multiplier: "439460" - , summand: "155381" - } - , unlockStakeEpoch: "184467" - , updateImplicit: "10000" - , updateProposalThd: "100000" - , updateVoteThd: "100000" - } - ' --null-input > "$dir"/byron-protocol-params.json - - cli_args=( - ## Note that these parameters are irrelevant by now. - --genesis-output-dir "$dir"/byron - --protocol-parameters-file "$dir"/byron-protocol-params.json - --start-time "$system_start_epoch" - --k "$(jq '.genesis.parameter_k' "$profile_json")" - --protocol-magic "$(jq '.genesis.network_magic' "$profile_json")" - --n-poor-addresses 1 - --n-delegate-addresses 1 - --total-balance 300000 - --delegate-share 0.9 - --avvm-entry-count 0 - --avvm-entry-balance 0 - ) - rm -rf "$dir"/byron - - verbose "genesis" "$(colorise cardano-cli byron genesis genesis "${cli_args[@]}")" - cardano-cli byron genesis genesis "${cli_args[@]}" -} - -genesis-byron-modular() { - local system_start_epoch=$1 - local dir=$2 - local profile_json=$3 - - # nix wants absolute paths - profile_json=$(realpath "$profile_json") - - evaluate --profile "$profile_json" genesis.byron > "$dir"/byron-protocol-params.json - read -r -a args <<< "$(evaluate --profile "${profile_json}" genesis.byron-genesis-args | jq -r)" - - cli_args=( - --genesis-output-dir "$dir"/byron - --protocol-parameters-file "$dir"/byron-protocol-params.json - --start-time "$system_start_epoch" - "${args[@]}" - ) - rm -rf "$dir"/byron - - verbose "genesis" "$(colorise cardano-cli byron genesis genesis "${cli_args[@]}")" - cardano-cli byron genesis genesis "${cli_args[@]}" -} - -genesis-create-staked() { - local profile_json=$1 - local dir=$2 - - rm -rf "$dir"/{*-keys,byron,pools,nodes,*.json,*.params,*.version} - mkdir -p "$dir" - - genesis spec shelley "$profile_json" >"$dir/genesis.spec.json" - genesis spec alonzo "$profile_json" >"$dir/genesis.alonzo.spec.json" - genesis spec conway "$profile_json" >"$dir/genesis.conway.spec.json" - genesis pool-relays "$profile_json" >"$dir/pool-relays.json" - - read -r -a args <<<"$(jq --raw-output '.cli_args.createStakedArgs | join(" ")' "$profile_json")" - create_staked_args=( - --genesis-dir "$dir" - --relay-specification-file "$dir/pool-relays.json" - "${args[@]}" - ) - progress "genesis" "$(colorise cardano-cli genesis create-staked "${create_staked_args[@]}")" - cardano-cli genesis create-staked "${create_staked_args[@]}" - - mv "$dir"/genesis.json "$dir"/genesis-shelley.json - mv "$dir"/genesis.spec.json "$dir"/genesis-shelley.spec.json - - info genesis "removing delegator keys." - rm "$dir"/stake-delegator-keys -rf - - info genesis "moving keys" - Massage_the_key_file_layout_to_match_AWS "$profile_json" "$node_specs" "$dir" -} - -genesis-create-testnet-data() { - local profile_json=$1 - local dir=$2 - - mkdir -p "$dir" - - genesis spec shelley "$profile_json" >"$dir/shelley-genesis.spec.json" - genesis spec alonzo "$profile_json" >"$dir/alonzo-genesis.spec.json" - genesis spec conway "$profile_json" >"$dir/conway-genesis.spec.json" - genesis pool-relays "$profile_json" >"$dir/pool-relays.json" - - local era - era=$(jq --raw-output '.era' "$profile_json") - - # TODO if profile_json.composition.dense_pool_density != 1 -> create-testnet-data does not support dense pools - read -r -a args <<<"$(jq --raw-output '.cli_args.createTestnetDataArgs | join(" ")' "$profile_json")" - create_testnet_data_args=( - --spec-shelley "$dir/shelley-genesis.spec.json" - --spec-alonzo "$dir/alonzo-genesis.spec.json" - --spec-conway "$dir/conway-genesis.spec.json" - --relays "$dir/pool-relays.json" - --out-dir "$dir" - "${args[@]}" - ) - progress genesis "$(colorise cardano-cli "$era" create-testnet-data "${create_testnet_data_args[@]}")" - cardano-cli "$era" genesis create-testnet-data "${create_testnet_data_args[@]}" - - # for comparison/compatibility - - # genesis specs and finals - jq_fmutate "$dir/shelley-genesis.spec.json" -S . - jq_fmutate "$dir/shelley-genesis.json" -S . - jq_fmutate "$dir/alonzo-genesis.spec.json" -S . - jq_fmutate "$dir/alonzo-genesis.json" -S . - jq_fmutate "$dir/conway-genesis.spec.json" -S . - jq_fmutate "$dir/conway-genesis.json" -S . - jq_fmutate "$dir/dijkstra-genesis.json" -S . - - ln -sf shelley-genesis.spec.json "$dir/genesis-shelley.spec.json" - ln -sf shelley-genesis.json "$dir/genesis-shelley.json" - ln -sf alonzo-genesis.spec.json "$dir/genesis.alonzo.spec.json" - ln -sf alonzo-genesis.json "$dir/genesis.alonzo.json" - ln -sf conway-genesis.spec.json "$dir/genesis.conway.spec.json" - ln -sf conway-genesis.json "$dir/genesis.conway.json" - ln -sf dijkstra-genesis.json "$dir/genesis.dijkstra.json" - - shopt -s extglob - - function link_keys() { - local from=${1} - local to=${2} - - for k in "$dir/$from"/*/*; do - local path=${k//"$dir/"/} - local elems - IFS=/ read -a elems -r <<<"$path" - local no=${elems[1]##*([[:alpha:]])} - local keyname=${elems[2]%.*} - local keyext=${elems[2]##*.} - ln -sf "../${path}" "$dir/${to}/${keyname}${no}.${keyext}" - done - } - - mkdir -p "$dir/pools" - link_keys pools-keys pools - - mkdir -p "$dir/utxo-keys" - link_keys utxo-keys utxo-keys - - local is_voting - is_voting=$(jq --raw-output '.workloads | any( .name == "voting")' "$profile_json") - if [[ "$is_voting" == "true" ]]; - then - info genesis "voting workload specified - keeping one stake key per producer" - mv "$dir/stake-delegators" "$dir/stake-delegators.bak" - mkdir "$dir/stake-delegators" - local pools - pools="$(jq --raw-output '.composition.n_pools' "${profile_json}")" - for i in $(seq 1 "$pools") - do - if test -d "$dir/stake-delegators.bak/delegator${i}" - then - local from_dir to_dir - from_dir="$dir/stake-delegators.bak/delegator${i}" - to_dir="$dir/stake-delegators/delegator$((i - 1))" - mkdir "$to_dir" - cp "$from_dir"/{payment,staking}.{skey,vkey} "$to_dir"/ - fi - done - rm "$dir/stake-delegators.bak" -rf - info genesis "voting workload specified - skipping deletion of DRep keys" - else - info genesis "removing delegator keys." - rm "$dir/stake-delegators" -rf - info genesis "removing dreps keys." - rm "$dir"/drep-keys -rf - fi - - info genesis "moving keys" - Massage_the_key_file_layout_to_match_AWS "$profile_json" "$node_specs" "$dir" - - info genesis "create-testnet-data genesis available in $dir" -} diff --git a/nix/workbench/profile/presets/zero/README.md b/nix/workbench/profile/presets/zero/README.md new file mode 100644 index 00000000000..0cb1b01cc51 --- /dev/null +++ b/nix/workbench/profile/presets/zero/README.md @@ -0,0 +1,156 @@ +Used to create and cache the dataset, what is passed to `create-testnet-data` arguments `--spec-shelley`, `--spec-alonzo` and `--spec-conway`. + +``` +cardano-cli latest genesis create-testnet-data --spec-shelley genesis/shelley-genesis.json --spec-alonzo genesis/alonzo-genesis.json --spec-conway genesis/conway-genesis.json --genesis-keys 0 --pools 0 --stake-delegators 0 --committee-keys 0 --drep-keys 0 --stuffed-utxo 0 --utxo-keys 0 --total-supply 0 --delegated-supply 0 --testnet-magic 42 --start-time 1970-01-01T00:00:00Z +``` + +# What each parameter controls + +## Always-created files (regardless of parameters): + +- `shelley-genesis.json` +- `byron-genesis.json` +- `byron.genesis.spec.json` +- `alonzo-genesis.json` +- `conway-genesis.json` +- `dijkstra-genesis.json` +- `byron-gen-command/` (directory, always present) + +## When each parameter takes effect + +| Parameter | Activation | +|----------------------|---------------------------------------------------------------------------| +| `--total-supply` | >= 1 | +| `--delegated-supply` | >= 1 AND (`utxo-keys >= 1` OR (`pools >= 1` AND `stake-delegators >= 1`)) | +| `--utxo-keys` | >= 1 | +| `--pools` | >= 2 (1 entry always exists by default) | +| `--stake-delegators` | >= 1 (keys always created; JSON fields only when `pools >= 1`) | +| `--drep-keys` | >= 1 | +| `--stuffed-utxo` | >= 1 | +| `--genesis-keys` | (unused by us, always 0) | +| `--committee-keys` | (unused by us, always 0) | +| `--testnet-magic` | always 42 | + +## What each parameter changes in JSON files + +### Always-present, unchanged files + +- `alonzo-genesis.json` +- `dijkstra-genesis.json` + +### Shelley and Conway + +| Parameter | shelley-genesis.json | conway-genesis.json | +|----------------------|--------------------------------------------------------------------------------|---------------------| +| `--total-supply` | `maxLovelaceSupply`, `initialFunds` values | -- | +| `--delegated-supply` | `initialFunds` values (splits lovelace between utxo and staked entries) | -- | +| `--utxo-keys` | `initialFunds` (adds enterprise address entries) | -- | +| `--stake-delegators` | `initialFunds` (adds staked address entries), `staking.stake`, `staking.pools` | -- | +| `--drep-keys` | -- | `initialDReps` | +| `--stuffed-utxo` | `initialFunds` (adds entries with 0 lovelace) | -- | +| `--testnet-magic` | `networkMagic` | -- | + +## Byron (byron-genesis.json) + +Only `--pools` affects `byron-genesis.json`. No other parameter changes it. + +The three fields `bootStakeholders`, `heavyDelegation` and `nonAvvmBalances` +always have at least 1 entry (a default BFT entry), even with `--pools 0`. +Setting `--pools 1` produces the same structure as `--pools 0`. Additional +entries appear starting at `--pools 2`. + +| pools | `bootStakeholders` | `heavyDelegation` | `nonAvvmBalances` | +|-------|---------------------------|-------------------|-------------------| +| 0 | 1 entry (weight 1) | 1 entry | 1 x 3,000,000,000 | +| 1 | 1 entry (weight 1) | 1 entry | 1 x 3,000,000,000 | +| 5 | 5 entries (weight 1 each) | 5 entries | 5 x 3,000,000,000 | + +The value in `nonAvvmBalances` is always **3,000,000,000 per entry**, hardcoded. +It does not change with `--total-supply`, `--delegated-supply`, or any other +parameter. This is Byron-era lovelace in a separate ledger from Shelley's +`initialFunds`. + +All cryptographic material in these three fields (key hashes, public keys, +delegation certs, and Byron addresses) is **randomly generated on every run**. +Two calls with identical parameters will produce different keys and addresses +but the same structure and the same monetary values. + +## What files and directories each parameter creates + +| Parameter | Directory created | Contents per entry | +|---|---|---| +| `--pools` | `pools-keys/poolN/` | `byron-delegate.key`, `byron-delegation.cert`, `cold.skey`, `cold.vkey`, `kes.skey`, `kes.vkey`, `opcert.cert`, `opcert.counter`, `staking-reward.skey`, `staking-reward.vkey`, `vrf.skey`, `vrf.vkey` (12 files) | +| `--stake-delegators` | `stake-delegators/delegatorN/` | `payment.skey`, `payment.vkey`, `staking.skey`, `staking.vkey` (4 files) | +| `--drep-keys` | `drep-keys/drepN/` | `drep.skey`, `drep.vkey` (2 files) | +| `--utxo-keys` | `utxo-keys/utxoN/` | `utxo.skey`, `utxo.vkey` (2 files) | +| `--pools` | `byron-gen-command/` | `genesis-keys.NNN.key` (1 per pool) | +| (always) | `byron-gen-command/` | When pools=0: `delegate-keys.000.key`, `delegation-cert.000.json`, `genesis-keys.000.key` (default BFT entry) | + +# Funds distribution + +`--total-supply` sets `maxLovelaceSupply` in shelley-genesis.json. The actual +lovelace distributed in `initialFunds` is less: 10% of each supply slice goes to +Shelley reserves (the pot from which staking rewards are paid). + +``` +reserves = total-supply - utxo_total - staked_total +``` + +## `--utxo-keys` + +``` +non_delegated = total-supply - delegated-supply +utxo_total = non_delegated - floor(non_delegated / 10) +utxo_each = floor(utxo_total / utxo-keys) +``` + +## `--staked-delegators` + +``` +staked_total = delegated-supply - floor(delegated-supply / 10) +staked_each = floor(staked_total / stake-delegators) +``` + +UTxO key addresses use enterprise address format (prefix `60`). +Staked delegator addresses use staked address format (prefix `00`). + +`--delegated-supply` controls how the supply is split between these two groups. +Without it (or set to 0), the full supply (minus reserves) goes to UTxO keys. +It requires `--utxo-keys >= 1` or (`--pools >= 1` AND `--stake-delegators >= 1`) +to have any visible effect. + +`staking.pools` in shelley-genesis.json = min(pools, stake-delegators). +`staking.stake` = stake-delegators (when pools >= 1), otherwise 0. + +Example with `--total-supply 1000000 --delegated-supply 500000` (`maxLovelaceSupply` = 1,000,000): + +| utxo-keys | pools | stake | utxo total | staked total | initialFunds | utxo each | staked each | +|-----------|-------|-------|------------|--------------|--------------|-----------|-------------| +| 0 | 0 | 0 | 0 | 0 | 0 | -- | -- | +| 0 | 0 | 1 | 0 | 0 | 0 | -- | -- | +| 0 | 0 | 10 | 0 | 0 | 0 | -- | -- | +| 0 | 1 | 0 | 0 | 0 | 0 | -- | -- | +| 0 | 1 | 1 | 0 | 450,000 | 450,000 | -- | 450,000 | +| 0 | 1 | 10 | 0 | 450,000 | 450,000 | -- | 45,000 | +| 0 | 5 | 0 | 0 | 0 | 0 | -- | -- | +| 0 | 5 | 1 | 0 | 450,000 | 450,000 | -- | 450,000 | +| 0 | 5 | 10 | 0 | 450,000 | 450,000 | -- | 45,000 | +| 1 | 0 | 0 | 450,000 | 0 | 450,000 | 450,000 | -- | +| 1 | 0 | 1 | 450,000 | 0 | 450,000 | 450,000 | -- | +| 1 | 0 | 10 | 450,000 | 0 | 450,000 | 450,000 | -- | +| 1 | 1 | 0 | 450,000 | 0 | 450,000 | 450,000 | -- | +| 1 | 1 | 1 | 450,000 | 450,000 | 900,000 | 450,000 | 450,000 | +| 1 | 1 | 10 | 450,000 | 450,000 | 900,000 | 450,000 | 45,000 | +| 1 | 5 | 0 | 450,000 | 0 | 450,000 | 450,000 | -- | +| 1 | 5 | 1 | 450,000 | 450,000 | 900,000 | 450,000 | 450,000 | +| 1 | 5 | 10 | 450,000 | 450,000 | 900,000 | 450,000 | 45,000 | +| 2 | 0 | 0 | 450,000 | 0 | 450,000 | 225,000 | -- | +| 2 | 0 | 1 | 450,000 | 0 | 450,000 | 225,000 | -- | +| 2 | 0 | 10 | 450,000 | 0 | 450,000 | 225,000 | -- | +| 2 | 1 | 0 | 450,000 | 0 | 450,000 | 225,000 | -- | +| 2 | 1 | 1 | 450,000 | 450,000 | 900,000 | 225,000 | 450,000 | +| 2 | 1 | 10 | 450,000 | 450,000 | 900,000 | 225,000 | 45,000 | +| 2 | 5 | 0 | 450,000 | 0 | 450,000 | 225,000 | -- | +| 2 | 5 | 1 | 450,000 | 450,000 | 900,000 | 225,000 | 450,000 | +| 2 | 5 | 10 | 450,000 | 450,000 | 900,000 | 225,000 | 45,000 | + diff --git a/nix/workbench/profile/presets/zero/generate-all-combos.sh b/nix/workbench/profile/presets/zero/generate-all-combos.sh new file mode 100755 index 00000000000..daf8696c416 --- /dev/null +++ b/nix/workbench/profile/presets/zero/generate-all-combos.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CARDANO_CLI="./result/bin/cardano-cli" +SPEC_DIR="out-dir/zero" +OUT_BASE="out-dir" + +TOTAL_SUPPLY=(0 1 1000000) +DELEGATED_SUPPLY=(0 1 500000) +UTXO_KEYS=(0 1 2) +POOLS=(0 1 5) +STAKE=(0 1 10) +DREP_KEYS=(0 1 2) +STUFFED=(0 1 10) + +count=0 +skipped=0 + +for total in "${TOTAL_SUPPLY[@]}"; do + for delegated in "${DELEGATED_SUPPLY[@]}"; do + if (( delegated > total )); then + continue + fi + for utxo in "${UTXO_KEYS[@]}"; do + for pools in "${POOLS[@]}"; do + for stake in "${STAKE[@]}"; do + for drep in "${DREP_KEYS[@]}"; do + for stuffed in "${STUFFED[@]}"; do + + name="total-${total}-delegated-${delegated}-utxo-${utxo}-pools-${pools}-stake-${stake}-drep-${drep}-stuffed-${stuffed}" + outdir="${OUT_BASE}/${name}" + + if [[ -d "$outdir" ]]; then + skipped=$((skipped + 1)) + continue + fi + + mkdir -p "$outdir" + + if ! "$CARDANO_CLI" latest genesis create-testnet-data \ + --out-dir "$outdir" \ + --spec-shelley "$SPEC_DIR/shelley-genesis.json" \ + --spec-alonzo "$SPEC_DIR/alonzo-genesis.json" \ + --spec-conway "$SPEC_DIR/conway-genesis.json" \ + --genesis-keys 0 \ + --pools "$pools" \ + --stake-delegators "$stake" \ + --committee-keys 0 \ + --drep-keys "$drep" \ + --stuffed-utxo "$stuffed" \ + --utxo-keys "$utxo" \ + --total-supply "$total" \ + --delegated-supply "$delegated" \ + --testnet-magic 42 \ + --start-time 1970-01-01T00:00:00Z \ + 2>"$outdir/_stderr.txt"; then + echo "FAIL: $name (exit $?)" + cat "$outdir/_stderr.txt" + else + count=$((count + 1)) + echo "OK: $name" + fi + + done + done + done + done + done + done +done + +echo "" +echo "Done: $count generated, $skipped skipped (already existed)" + diff --git a/nix/workbench/profile/presets/zero/genesis/README.md b/nix/workbench/profile/presets/zero/genesis/README.md new file mode 100644 index 00000000000..dfb19b63243 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/README.md @@ -0,0 +1,6 @@ +Obtained files using the base and all other parameters set to zero + +``` +cardano-cli latest genesis create-testnet-data --spec-shelley base/shelley-genesis.json --spec-alonzo base/alonzo-genesis.json --spec-conway base/conway-genesis.json --genesis-keys 0 --pools 0 --stake-delegators 0 --committee-keys 0 --drep-keys 0 --stuffed-utxo 0 --utxo-keys 0 --total-supply 0 --delegated-supply 0 --testnet-magic 42 --start-time 1970-01-01T00:00:00Z +``` + diff --git a/nix/workbench/profile/presets/zero/genesis/alonzo-genesis.json b/nix/workbench/profile/presets/zero/genesis/alonzo-genesis.json new file mode 100644 index 00000000000..1d1ebbf552e --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/alonzo-genesis.json @@ -0,0 +1,188 @@ +{ + "collateralPercentage": 0, + "costModels": { + "PlutusV1": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "executionPrices": { + "priceMemory": 0, + "priceSteps": 0 + }, + "lovelacePerUTxOWord": 0, + "maxBlockExUnits": { + "memory": 0, + "steps": 0 + }, + "maxCollateralInputs": 0, + "maxTxExUnits": { + "memory": 0, + "steps": 0 + }, + "maxValueSize": 0 +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/base/README.md b/nix/workbench/profile/presets/zero/genesis/base/README.md new file mode 100644 index 00000000000..9853c64cb77 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/base/README.md @@ -0,0 +1,7 @@ +Files from mainnet with all its values set to zero (as much as cardano-cli parsing allows) + +- https://book.world.dev.cardano.org/environments/mainnet/byron-genesis.json +- https://book.world.dev.cardano.org/environments/mainnet/shelley-genesis.json +- https://book.world.dev.cardano.org/environments/mainnet/alonzo-genesis.json +- https://book.world.dev.cardano.org/environments/mainnet/conway-genesis.json + diff --git a/nix/workbench/profile/presets/zero/genesis/base/alonzo-genesis.json b/nix/workbench/profile/presets/zero/genesis/base/alonzo-genesis.json new file mode 100644 index 00000000000..94f7adc5f57 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/base/alonzo-genesis.json @@ -0,0 +1,196 @@ +{ + "lovelacePerUTxOWord": 0, + "executionPrices": { + "prSteps": + { + "numerator" : 0, + "denominator" : 1 + }, + "prMem": + { + "numerator" : 0, + "denominator" : 1 + } + }, + "maxTxExUnits": { + "exUnitsMem": 0, + "exUnitsSteps": 0 + }, + "maxBlockExUnits": { + "exUnitsMem": 0, + "exUnitsSteps": 0 + }, + "maxValueSize": 0, + "collateralPercentage": 0, + "maxCollateralInputs": 0, + "costModels": { + "PlutusV1": { + "sha2_256-memory-arguments": 0, + "equalsString-cpu-arguments-constant": 0, + "cekDelayCost-exBudgetMemory": 0, + "lessThanEqualsByteString-cpu-arguments-intercept": 0, + "divideInteger-memory-arguments-minimum": 0, + "appendByteString-cpu-arguments-slope": 0, + "blake2b-cpu-arguments-slope": 0, + "iData-cpu-arguments": 0, + "encodeUtf8-cpu-arguments-slope": 0, + "unBData-cpu-arguments": 0, + "multiplyInteger-cpu-arguments-intercept": 0, + "cekConstCost-exBudgetMemory": 0, + "nullList-cpu-arguments": 0, + "equalsString-cpu-arguments-intercept": 0, + "trace-cpu-arguments": 0, + "mkNilData-memory-arguments": 0, + "lengthOfByteString-cpu-arguments": 0, + "cekBuiltinCost-exBudgetCPU": 0, + "bData-cpu-arguments": 0, + "subtractInteger-cpu-arguments-slope": 0, + "unIData-cpu-arguments": 0, + "consByteString-memory-arguments-intercept": 0, + "divideInteger-memory-arguments-slope": 0, + "divideInteger-cpu-arguments-model-arguments-slope": 0, + "listData-cpu-arguments": 0, + "headList-cpu-arguments": 0, + "chooseData-memory-arguments": 0, + "equalsInteger-cpu-arguments-intercept": 0, + "sha3_256-cpu-arguments-slope": 0, + "sliceByteString-cpu-arguments-slope": 0, + "unMapData-cpu-arguments": 0, + "lessThanInteger-cpu-arguments-intercept": 0, + "mkCons-cpu-arguments": 0, + "appendString-memory-arguments-intercept": 0, + "modInteger-cpu-arguments-model-arguments-slope": 0, + "ifThenElse-cpu-arguments": 0, + "mkNilPairData-cpu-arguments": 0, + "lessThanEqualsInteger-cpu-arguments-intercept": 0, + "addInteger-memory-arguments-slope": 0, + "chooseList-memory-arguments": 0, + "constrData-memory-arguments": 0, + "decodeUtf8-cpu-arguments-intercept": 0, + "equalsData-memory-arguments": 0, + "subtractInteger-memory-arguments-slope": 0, + "appendByteString-memory-arguments-intercept": 0, + "lengthOfByteString-memory-arguments": 0, + "headList-memory-arguments": 0, + "listData-memory-arguments": 0, + "consByteString-cpu-arguments-intercept": 0, + "unIData-memory-arguments": 0, + "remainderInteger-memory-arguments-minimum": 0, + "bData-memory-arguments": 0, + "lessThanByteString-cpu-arguments-slope": 0, + "encodeUtf8-memory-arguments-intercept": 0, + "cekStartupCost-exBudgetCPU": 0, + "multiplyInteger-memory-arguments-intercept": 0, + "unListData-memory-arguments": 0, + "remainderInteger-cpu-arguments-model-arguments-slope": 0, + "cekVarCost-exBudgetCPU": 0, + "remainderInteger-memory-arguments-slope": 0, + "cekForceCost-exBudgetCPU": 0, + "sha2_256-cpu-arguments-slope": 0, + "equalsInteger-memory-arguments": 0, + "indexByteString-memory-arguments": 0, + "addInteger-memory-arguments-intercept": 0, + "chooseUnit-cpu-arguments": 0, + "sndPair-cpu-arguments": 0, + "cekLamCost-exBudgetCPU": 0, + "fstPair-cpu-arguments": 0, + "quotientInteger-memory-arguments-minimum": 0, + "decodeUtf8-cpu-arguments-slope": 0, + "lessThanInteger-memory-arguments": 0, + "lessThanEqualsInteger-cpu-arguments-slope": 0, + "fstPair-memory-arguments": 0, + "modInteger-memory-arguments-intercept": 0, + "unConstrData-cpu-arguments": 0, + "lessThanEqualsInteger-memory-arguments": 0, + "chooseUnit-memory-arguments": 0, + "sndPair-memory-arguments": 0, + "addInteger-cpu-arguments-intercept": 0, + "decodeUtf8-memory-arguments-slope": 0, + "equalsData-cpu-arguments-intercept": 0, + "mapData-cpu-arguments": 0, + "mkPairData-cpu-arguments": 0, + "quotientInteger-cpu-arguments-constant": 0, + "consByteString-memory-arguments-slope": 0, + "cekVarCost-exBudgetMemory": 0, + "indexByteString-cpu-arguments": 0, + "unListData-cpu-arguments": 0, + "equalsInteger-cpu-arguments-slope": 0, + "cekStartupCost-exBudgetMemory": 0, + "subtractInteger-cpu-arguments-intercept": 0, + "divideInteger-cpu-arguments-model-arguments-intercept": 0, + "divideInteger-memory-arguments-intercept": 0, + "cekForceCost-exBudgetMemory": 0, + "blake2b-cpu-arguments-intercept": 0, + "remainderInteger-cpu-arguments-constant": 0, + "tailList-cpu-arguments": 0, + "encodeUtf8-cpu-arguments-intercept": 0, + "equalsString-cpu-arguments-slope": 0, + "lessThanByteString-memory-arguments": 0, + "multiplyInteger-cpu-arguments-slope": 0, + "appendByteString-cpu-arguments-intercept": 0, + "lessThanEqualsByteString-cpu-arguments-slope": 0, + "modInteger-memory-arguments-slope": 0, + "addInteger-cpu-arguments-slope": 0, + "equalsData-cpu-arguments-slope": 0, + "decodeUtf8-memory-arguments-intercept": 0, + "chooseList-cpu-arguments": 0, + "constrData-cpu-arguments": 0, + "equalsByteString-memory-arguments": 0, + "cekApplyCost-exBudgetCPU": 0, + "quotientInteger-memory-arguments-slope": 0, + "verifySignature-cpu-arguments-intercept": 0, + "unMapData-memory-arguments": 0, + "mkCons-memory-arguments": 0, + "sliceByteString-memory-arguments-slope": 0, + "sha3_256-memory-arguments": 0, + "ifThenElse-memory-arguments": 0, + "mkNilPairData-memory-arguments": 0, + "equalsByteString-cpu-arguments-slope": 0, + "appendString-cpu-arguments-intercept": 0, + "quotientInteger-cpu-arguments-model-arguments-slope": 0, + "cekApplyCost-exBudgetMemory": 0, + "equalsString-memory-arguments": 0, + "multiplyInteger-memory-arguments-slope": 0, + "cekBuiltinCost-exBudgetMemory": 0, + "remainderInteger-memory-arguments-intercept": 0, + "sha2_256-cpu-arguments-intercept": 0, + "remainderInteger-cpu-arguments-model-arguments-intercept": 0, + "lessThanEqualsByteString-memory-arguments": 0, + "tailList-memory-arguments": 0, + "mkNilData-cpu-arguments": 0, + "chooseData-cpu-arguments": 0, + "unBData-memory-arguments": 0, + "blake2b-memory-arguments": 0, + "iData-memory-arguments": 0, + "nullList-memory-arguments": 0, + "cekDelayCost-exBudgetCPU": 0, + "subtractInteger-memory-arguments-intercept": 0, + "lessThanByteString-cpu-arguments-intercept": 0, + "consByteString-cpu-arguments-slope": 0, + "appendByteString-memory-arguments-slope": 0, + "trace-memory-arguments": 0, + "divideInteger-cpu-arguments-constant": 0, + "cekConstCost-exBudgetCPU": 0, + "encodeUtf8-memory-arguments-slope": 0, + "quotientInteger-cpu-arguments-model-arguments-intercept": 0, + "mapData-memory-arguments": 0, + "appendString-cpu-arguments-slope": 0, + "modInteger-cpu-arguments-constant": 0, + "verifySignature-cpu-arguments-slope": 0, + "unConstrData-memory-arguments": 0, + "quotientInteger-memory-arguments-intercept": 0, + "equalsByteString-cpu-arguments-constant": 0, + "sliceByteString-memory-arguments-intercept": 0, + "mkPairData-memory-arguments": 0, + "equalsByteString-cpu-arguments-intercept": 0, + "appendString-memory-arguments-slope": 0, + "lessThanInteger-cpu-arguments-slope": 0, + "modInteger-cpu-arguments-model-arguments-intercept": 0, + "modInteger-memory-arguments-minimum": 0, + "sha3_256-cpu-arguments-intercept": 0, + "verifySignature-memory-arguments": 0, + "cekLamCost-exBudgetMemory": 0, + "sliceByteString-cpu-arguments-intercept": 0 + } + } +} diff --git a/nix/workbench/profile/presets/zero/genesis/base/byron-genesis.json b/nix/workbench/profile/presets/zero/genesis/base/byron-genesis.json new file mode 100644 index 00000000000..7c693883308 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/base/byron-genesis.json @@ -0,0 +1,42 @@ +{ + "avvmDistr": { + }, + "blockVersionData": { + "heavyDelThd": "0", + "maxBlockSize": "0", + "maxHeaderSize": "0", + "maxProposalSize": "0", + "maxTxSize": "0", + "mpcThd": "0", + "scriptVersion": 0, + "slotDuration": "0", + "softforkRule": { + "initThd": "0", + "minThd": "0", + "thdDecrement": "0" + }, + "txFeePolicy": { + "multiplier": "0", + "summand": "0" + }, + "unlockStakeEpoch": "0", + "updateImplicit": "0", + "updateProposalThd": "0", + "updateVoteThd": "0" + }, + "ftsSeed": "0", + "protocolConsts": { + "k": 0, + "protocolMagic": 0, + "vssMaxTTL": 0, + "vssMinTTL": 0 + }, + "startTime": 0, + "bootStakeholders": { + }, + "heavyDelegation": { + }, + "nonAvvmBalances": {}, + "vssCerts": { + } +} diff --git a/nix/workbench/profile/presets/zero/genesis/base/conway-genesis.json b/nix/workbench/profile/presets/zero/genesis/base/conway-genesis.json new file mode 100644 index 00000000000..df38f46f1d0 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/base/conway-genesis.json @@ -0,0 +1,296 @@ +{ + "poolVotingThresholds": { + "committeeNormal": 0, + "committeeNoConfidence": 0, + "hardForkInitiation": 0, + "motionNoConfidence": 0, + "ppSecurityGroup": 0 + }, + "dRepVotingThresholds": { + "motionNoConfidence": 0, + "committeeNormal": 0, + "committeeNoConfidence": 0, + "updateToConstitution": 0, + "hardForkInitiation": 0, + "ppNetworkGroup": 0, + "ppEconomicGroup": 0, + "ppTechnicalGroup": 0, + "ppGovGroup": 0, + "treasuryWithdrawal": 0 + }, + "committeeMinSize": 0, + "committeeMaxTermLength": 0, + "govActionLifetime": 0, + "govActionDeposit": 0, + "dRepDeposit": 0, + "dRepActivity": 0, + "minFeeRefScriptCostPerByte": 0, + "plutusV3CostModel": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "constitution": { + "anchor": { + "dataHash": "ca41a91f399259bcefe57f9858e91f6d00e1a38d6d9c63d4052914ea7bd70cb2", + "url": "ipfs://bafkreifnwj6zpu3ixa4siz2lndqybyc5wnnt3jkwyutci4e2tmbnj3xrdm" + }, + "script": "fa24fb305126805cf2164c161d852a0e7330cf988f1fe558cf7d4a64" + }, + "committee": { + "members": { + }, + "threshold": { + "numerator": 0, + "denominator": 1 + } + } +} diff --git a/nix/workbench/profile/presets/zero/genesis/base/shelley-genesis.json b/nix/workbench/profile/presets/zero/genesis/base/shelley-genesis.json new file mode 100644 index 00000000000..70b4a8a11e6 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/base/shelley-genesis.json @@ -0,0 +1,40 @@ +{ + "activeSlotsCoeff": 0.000000001, + "protocolParams": { + "protocolVersion": { + "minor": 0, + "major": 0 + }, + "decentralisationParam": 0, + "eMax": 0, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "maxTxSize": 0, + "maxBlockBodySize": 0, + "maxBlockHeaderSize": 0, + "minFeeA": 0, + "minFeeB": 0, + "minUTxOValue": 0, + "poolDeposit": 0, + "minPoolCost": 0, + "keyDeposit": 0, + "nOpt": 0, + "rho": 0, + "tau": 0, + "a0": 0 + }, + "genDelegs": { + }, + "updateQuorum": 0, + "networkId": "Mainnet", + "initialFunds": {}, + "maxLovelaceSupply": 0, + "networkMagic": 0, + "epochLength": 0, + "systemStart": "2017-09-23T21:44:51Z", + "slotsPerKESPeriod": 0, + "slotLength": 0, + "maxKESEvolutions": 0, + "securityParam": 1 +} diff --git a/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegate-keys.000.key b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegate-keys.000.key new file mode 100644 index 00000000000..8e198bb5df3 Binary files /dev/null and b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegate-keys.000.key differ diff --git a/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegation-cert.000.json b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegation-cert.000.json new file mode 100644 index 00000000000..f946fd47cb6 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/delegation-cert.000.json @@ -0,0 +1,8 @@ +{ "omega": 0 +, "issuerPk": + "wnxv0Hr7nriM2t2qzFJK6ocqiWKrv7byxzMPwghKVZdc4fcudeW53PK/fukjrCcuYdv5Il7O+9eGpSfei5LFlg==" +, "delegatePk": + "8CSVrRYQrHu5wbggti3Ly6Gl2xKNIczbwZ/EXwE5PAiG6A/PoyG3Xf4WuC3JP4yOAHuCyRyGobXgdnEV2maPEg==" +, "cert": + "0b791623a7e3bf1eb0ef6df6b15a3ecde519e117ce972c4d136fbb482e3fed38a38eeebd5e79959b62bdc30b9eb537adfa8e313b1729cf1136a47bd794213500" +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/byron-gen-command/genesis-keys.000.key b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/genesis-keys.000.key new file mode 100644 index 00000000000..daece5375ff --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/byron-gen-command/genesis-keys.000.key @@ -0,0 +1 @@ +X0k;$Ʃ9"<4!X8bD&LAN{%>V2;0ChEU#X|ozݪRJ*b3JU\.u~#'.a"^׆'ދŖ \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/byron-genesis.json b/nix/workbench/profile/presets/zero/genesis/byron-genesis.json new file mode 100644 index 00000000000..53d667c6564 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/byron-genesis.json @@ -0,0 +1,40 @@ +{ "bootStakeholders": + { "701a09650e6afc1d40ab00b852b2d1510562c55cb9bcb3f47bc8243e": 1 } +, "heavyDelegation": + { "701a09650e6afc1d40ab00b852b2d1510562c55cb9bcb3f47bc8243e": + { "omega": 0 + , "issuerPk": + "wnxv0Hr7nriM2t2qzFJK6ocqiWKrv7byxzMPwghKVZdc4fcudeW53PK/fukjrCcuYdv5Il7O+9eGpSfei5LFlg==" + , "delegatePk": + "8CSVrRYQrHu5wbggti3Ly6Gl2xKNIczbwZ/EXwE5PAiG6A/PoyG3Xf4WuC3JP4yOAHuCyRyGobXgdnEV2maPEg==" + , "cert": + "0b791623a7e3bf1eb0ef6df6b15a3ecde519e117ce972c4d136fbb482e3fed38a38eeebd5e79959b62bdc30b9eb537adfa8e313b1729cf1136a47bd794213500" + } } +, "startTime": 0 +, "nonAvvmBalances": + { "2657WMsDfac62bP8st6eavd6D6drPGLSTUsTFTa5J49n3kmY4puG8v1kRr1CJEHKr": + "3000000000" } +, "blockVersionData": + { "scriptVersion": 0 + , "slotDuration": "1000" + , "maxBlockSize": "2000000" + , "maxHeaderSize": "2000000" + , "maxTxSize": "4096" + , "maxProposalSize": "700" + , "mpcThd": "20000000000000" + , "heavyDelThd": "300000000000" + , "updateVoteThd": "1000000000000" + , "updateProposalThd": "100000000000000" + , "updateImplicit": "10000" + , "softforkRule": + { "initThd": "900000000000000" + , "minThd": "600000000000000" + , "thdDecrement": "50000000000000" + } + , "txFeePolicy": + { "summand": "155381000000000" , "multiplier": "43946000000" } + , "unlockStakeEpoch": "18446744073709551615" + } +, "protocolConsts": { "k": 1 , "protocolMagic": 42 } +, "avvmDistr": {} +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/byron.genesis.spec.json b/nix/workbench/profile/presets/zero/genesis/byron.genesis.spec.json new file mode 100644 index 00000000000..fba4d0ca89d --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/byron.genesis.spec.json @@ -0,0 +1,23 @@ +{ + "heavyDelThd": "300000000000", + "maxBlockSize": "2000000", + "maxHeaderSize": "2000000", + "maxProposalSize": "700", + "maxTxSize": "4096", + "mpcThd": "20000000000000", + "scriptVersion": 0, + "slotDuration": "1000", + "softforkRule": { + "initThd": "900000000000000", + "minThd": "600000000000000", + "thdDecrement": "50000000000000" + }, + "txFeePolicy": { + "multiplier": "43946000000", + "summand": "155381000000000" + }, + "unlockStakeEpoch": "18446744073709551615", + "updateImplicit": "10000", + "updateProposalThd": "100000000000000", + "updateVoteThd": "1000000000000" +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/conway-genesis.json b/nix/workbench/profile/presets/zero/genesis/conway-genesis.json new file mode 100644 index 00000000000..68cbeb0d7b6 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/conway-genesis.json @@ -0,0 +1,292 @@ +{ + "committee": { + "members": {}, + "threshold": 0 + }, + "committeeMaxTermLength": 0, + "committeeMinSize": 0, + "constitution": { + "anchor": { + "dataHash": "ca41a91f399259bcefe57f9858e91f6d00e1a38d6d9c63d4052914ea7bd70cb2", + "url": "ipfs://bafkreifnwj6zpu3ixa4siz2lndqybyc5wnnt3jkwyutci4e2tmbnj3xrdm" + }, + "script": "fa24fb305126805cf2164c161d852a0e7330cf988f1fe558cf7d4a64" + }, + "dRepActivity": 0, + "dRepDeposit": 0, + "dRepVotingThresholds": { + "committeeNoConfidence": 0, + "committeeNormal": 0, + "hardForkInitiation": 0, + "motionNoConfidence": 0, + "ppEconomicGroup": 0, + "ppGovGroup": 0, + "ppNetworkGroup": 0, + "ppTechnicalGroup": 0, + "treasuryWithdrawal": 0, + "updateToConstitution": 0 + }, + "govActionDeposit": 0, + "govActionLifetime": 0, + "minFeeRefScriptCostPerByte": 0, + "plutusV3CostModel": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "poolVotingThresholds": { + "committeeNoConfidence": 0, + "committeeNormal": 0, + "hardForkInitiation": 0, + "motionNoConfidence": 0, + "ppSecurityGroup": 0 + } +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/dijkstra-genesis.json b/nix/workbench/profile/presets/zero/genesis/dijkstra-genesis.json new file mode 100644 index 00000000000..4b117769447 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/dijkstra-genesis.json @@ -0,0 +1,6 @@ +{ + "maxRefScriptSizePerBlock": 1048576, + "maxRefScriptSizePerTx": 204800, + "refScriptCostMultiplier": 1.2, + "refScriptCostStride": 25600 +} \ No newline at end of file diff --git a/nix/workbench/profile/presets/zero/genesis/shelley-genesis.json b/nix/workbench/profile/presets/zero/genesis/shelley-genesis.json new file mode 100644 index 00000000000..9c9a50926a4 --- /dev/null +++ b/nix/workbench/profile/presets/zero/genesis/shelley-genesis.json @@ -0,0 +1,43 @@ +{ + "activeSlotsCoeff": 1.0e-9, + "epochLength": 0, + "genDelegs": {}, + "initialFunds": {}, + "maxKESEvolutions": 0, + "maxLovelaceSupply": 0, + "networkId": "Mainnet", + "networkMagic": 42, + "protocolParams": { + "a0": 0, + "decentralisationParam": 0, + "eMax": 0, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "keyDeposit": 0, + "maxBlockBodySize": 0, + "maxBlockHeaderSize": 0, + "maxTxSize": 0, + "minFeeA": 0, + "minFeeB": 0, + "minPoolCost": 0, + "minUTxOValue": 0, + "nOpt": 0, + "poolDeposit": 0, + "protocolVersion": { + "major": 0, + "minor": 0 + }, + "rho": 0, + "tau": 0 + }, + "securityParam": 1, + "slotLength": 0, + "slotsPerKESPeriod": 0, + "staking": { + "pools": {}, + "stake": {} + }, + "systemStart": "1970-01-01T00:00:00Z", + "updateQuorum": 0 +} \ No newline at end of file diff --git a/nix/workbench/run.sh b/nix/workbench/run.sh index 9d3935ba1cb..c48768f9a0c 100644 --- a/nix/workbench/run.sh +++ b/nix/workbench/run.sh @@ -263,9 +263,9 @@ EOF local run=${1:?$usage} local dir=$(run compute-path "$run") - if test ! -f "$dir"/genesis-shelley.json + if test ! -f "$dir"/genesis/genesis.shelley.json -a -f "$dir"/genesis.json then msg "fixing up genesis naming in: $dir" - mv "$dir"/genesis.json "$dir"/genesis-shelley.json; fi + mv "$dir"/genesis.json "$dir"/genesis/genesis.shelley.json; fi if test -z "$(ls -d "$dir"/node-* 2>/dev/null)" then msg "fixing up a legacy cardano-ops run in: $dir" @@ -327,7 +327,7 @@ EOF then return 1 else fatal "run $run (at $dir) missing a file: meta.json"; fi; fi - test -f "$dir"/profile.json -a -f "$dir"/genesis-shelley.json || + test -f "$dir"/profile.json -a -f "$dir"/genesis/genesis.shelley.json || run fix-legacy-run-structure "$run";; get-path | get ) @@ -538,9 +538,6 @@ EOF "$genesis_cache_entry" \ "$dir"/genesis fi - ## Record geneses - cp "$dir"/genesis/genesis-shelley.json "$dir"/genesis-shelley.json - cp "$dir"/genesis/genesis.alonzo.json "$dir"/genesis.alonzo.json echo >&2 ## Add global_basedir Voltaire Plutus guardrails script cp "$global_basedir"/genesis/guardrails-script.plutus "$dir"/genesis/ @@ -654,7 +651,7 @@ EOF local usage="USAGE: wb run $op RUN" local run=${1:?$usage} local dir=$global_rundir/$run - local genesis="$dir"/genesis-shelley.json + local genesis="$dir"/genesis/genesis.shelley.json local geneses_orig_dir=$global_rundir/.geneses.orig local genesis_orig="$geneses_orig_dir"/$run.orig.json @@ -673,7 +670,7 @@ EOF .initialFunds = {} | .staking = {} ' "$genesis_orig"; fi - cp -f "$dir"/genesis-shelley.json "$dir"/genesis/genesis-shelley.json;; + ;; package | pack ) local usage="USAGE: wb run $op RUN" @@ -812,8 +809,8 @@ run_remote_get() { jq . <<<$meta > $dir/meta.json local common_run_files=( - genesis.alonzo.json - genesis-shelley.json + genesis/genesis.alonzo.json + genesis/genesis.shelley.json profile.json generator/protocol-parameters-queried.json generator/plutus-budget-summary.json diff --git a/nix/workbench/service/generator.nix b/nix/workbench/service/generator.nix index 2f4a294edc3..5e218f7b199 100644 --- a/nix/workbench/service/generator.nix +++ b/nix/workbench/service/generator.nix @@ -40,7 +40,7 @@ let finaliseGeneratorService = profile: svc: recursiveUpdate svc ({ - sigKey = "../genesis/utxo-keys/utxo1.skey"; + sigKey = "../genesis/utxo-keys/utxo1/utxo.skey"; runScriptFile = "run-script.json"; ## path to the config and socket of the locally running node. nodeConfigFile = "../${runningNode}/config.json"; diff --git a/nix/workbench/service/nodes.nix b/nix/workbench/service/nodes.nix index 22e5a5f85cd..055b1176688 100644 --- a/nix/workbench/service/nodes.nix +++ b/nix/workbench/service/nodes.nix @@ -119,8 +119,8 @@ let TargetNumberOfActivePeers = max 15 valency; TargetNumberOfEstablishedPeers = max 40 valency; - ByronGenesisFile = "../genesis/byron/genesis.json"; - ShelleyGenesisFile = "../genesis/genesis-shelley.json"; + ByronGenesisFile = "../genesis/genesis.byron.json"; + ShelleyGenesisFile = "../genesis/genesis.shelley.json"; AlonzoGenesisFile = "../genesis/genesis.alonzo.json"; ConwayGenesisFile = "../genesis/genesis.conway.json"; DijkstraGenesisFile = "../genesis/genesis.dijkstra.json"; @@ -170,9 +170,12 @@ let executable = "cardano-node"; ######################################### } // optionalAttrs isProducer { - operationalCertificate = "../genesis/node-keys/node${toString i}.opcert"; - kesKey = "../genesis/node-keys/node-kes${toString i}.skey"; - vrfKey = "../genesis/node-keys/node-vrf${toString i}.skey"; + # Key paths match create-testnet-data output layout directly. + # create-testnet-data uses 1-indexed pool dirs (pool1, pool2, ...). + # Dense/bulk pools and BFT nodes are not supported by create-testnet-data. + operationalCertificate = "../genesis/pools-keys/pool${toString (i + 1)}/opcert.cert"; + kesKey = "../genesis/pools-keys/pool${toString (i + 1)}/kes.skey"; + vrfKey = "../genesis/pools-keys/pool${toString (i + 1)}/vrf.skey"; } // optionalAttrs profile.node.tracer { tracerSocketPathConnect = mkDefault "../tracer/tracer.socket"; }; diff --git a/nix/workbench/wb b/nix/workbench/wb index d4d6b9579a2..177237bf4fc 100755 --- a/nix/workbench/wb +++ b/nix/workbench/wb @@ -13,6 +13,9 @@ global_basedir=${global_basedir:-$(realpath "$(dirname "$0")")} # For genesis creating, create-testnet-data is the default CLI command; set to 0 to fall back to create-staked : "${WB_CREATE_TESTNET_DATA:=1}" +# Separate dataset and protocols caching using create-testnet-data; set to 0 to always run cardano-cli directly +: "${WB_GENESIS_CHUNKS:=0}" + # By default, enable the new database storage backend for `locli` : "${WB_LOCLI_DB:=1}" @@ -28,6 +31,9 @@ global_basedir=${global_basedir:-$(realpath "$(dirname "$0")")} . "$global_basedir"/analyse/analyse.sh . "$global_basedir"/profile/profile.sh . "$global_basedir"/genesis/genesis.sh +. "$global_basedir"/genesis/genesis-chunks.sh +. "$global_basedir"/genesis/genesis-jq.sh +. "$global_basedir"/genesis/genesis-modular.sh . "$global_basedir"/topology/topology.sh . "$global_basedir"/evaluate/evaluate.sh diff --git a/nix/workbench/workload/voting.nix b/nix/workbench/workload/voting.nix index b68f0c885cd..8b0063bb38d 100644 --- a/nix/workbench/workload/voting.nix +++ b/nix/workbench/workload/voting.nix @@ -26,8 +26,8 @@ let else throw "Conway genesis needed!" ; # Where to obtain the genesis funds from. - genesis_funds_vkey = "../../genesis/cache-entry/utxo-keys/utxo2.vkey"; - genesis_funds_skey = "../../genesis/cache-entry/utxo-keys/utxo2.skey"; + genesis_funds_vkey = "../../genesis/utxo-keys/utxo2/utxo.vkey"; + genesis_funds_skey = "../../genesis/utxo-keys/utxo2/utxo.skey"; # Initial donation from genesis funds to make "valid" withdrawal proposals. treasury_donation = 500000;