diff --git a/bench/budget_opt-20260422-190901/results.csv b/bench/budget_opt-20260422-190901/results.csv new file mode 100644 index 0000000000..1c7db94dfc --- /dev/null +++ b/bench/budget_opt-20260422-190901/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",268.75913299999957,293.12935219999963,318.0013615999997 +"soroswap,TX=2000,T=8",260.3968994999982,337.6564257000006,458.8638931899988 diff --git a/bench/budget_opt-20260422-190901/stamp b/bench/budget_opt-20260422-190901/stamp new file mode 100644 index 0000000000..12abb655ad --- /dev/null +++ b/bench/budget_opt-20260422-190901/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-155-g6add6c103-dirty of stellar-core +v26.0.0-155-g6add6c103-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.1 + git version: 9936a7086429401b69b3e0029d41ab9c22457312 + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/cache_budget-20260416-170151/results.csv b/bench/cache_budget-20260416-170151/results.csv new file mode 100644 index 0000000000..6e11280589 --- /dev/null +++ b/bench/cache_budget-20260416-170151/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",365.74236700000074,403.91725049999803,423.410349519993 +"soroswap,TX=2000,T=8",315.9667809999992,351.90522820000075,397.0369427499969 diff --git a/bench/cache_budget-20260416-170151/stamp b/bench/cache_budget-20260416-170151/stamp new file mode 100644 index 0000000000..57ca349d3e --- /dev/null +++ b/bench/cache_budget-20260416-170151/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-114-g56a7db07c-dirty of stellar-core +v26.0.0-114-g56a7db07c-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/cache_xdr_size-20260411-002309/results.csv b/bench/cache_xdr_size-20260411-002309/results.csv new file mode 100644 index 0000000000..41ecaef35a --- /dev/null +++ b/bench/cache_xdr_size-20260411-002309/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",316.87564600000405,342.05008529999867,358.4165310299993 +"sac,TX=3200,T=8",210.82947749999948,235.41631355000035,247.66588434000028 +"custom_token,TX=1600,T=1",294.94135599999936,323.27958394999735,335.3970549099996 +"custom_token,TX=1600,T=8",136.31600800000024,151.25250469999955,157.71860535000087 +"soroswap,TX=1000,T=1",449.36899600000106,481.23976025000013,509.2973847999979 +"soroswap,TX=1000,T=8",149.22892349999984,157.78476389999915,162.52508470000006 diff --git a/bench/cache_xdr_size-20260411-002309/stamp b/bench/cache_xdr_size-20260411-002309/stamp new file mode 100644 index 0000000000..349ee03797 --- /dev/null +++ b/bench/cache_xdr_size-20260411-002309/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-77-gf1c352b22-dirty of stellar-core +v26.0.0-77-gf1c352b22-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/create_upd_wo_loading-20260410-221400/results.csv b/bench/create_upd_wo_loading-20260410-221400/results.csv new file mode 100644 index 0000000000..bc4459e7e6 --- /dev/null +++ b/bench/create_upd_wo_loading-20260410-221400/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",322.9975675000005,348.8423988000006,380.21173682999887 +"sac,TX=3200,T=8",227.38690899999892,250.0006931499993,260.29676706999953 +"custom_token,TX=1600,T=1",308.1659649999997,330.4373707500009,347.41956414999964 +"custom_token,TX=1600,T=8",150.65184649999992,164.35087679999995,168.08713427000038 +"soroswap,TX=1000,T=1",477.5879510000086,539.8670865999983,585.6553417600018 +"soroswap,TX=1000,T=8",177.90089249999983,199.301519649996,209.90552744999786 diff --git a/bench/create_upd_wo_loading-20260410-221400/stamp b/bench/create_upd_wo_loading-20260410-221400/stamp new file mode 100644 index 0000000000..d0155ce56d --- /dev/null +++ b/bench/create_upd_wo_loading-20260410-221400/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-67-g541a82a14-dirty of stellar-core +v26.0.0-67-g541a82a14-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/disable_meta-20260410-205536/results.csv b/bench/disable_meta-20260410-205536/results.csv new file mode 100644 index 0000000000..ffbb9cd18c --- /dev/null +++ b/bench/disable_meta-20260410-205536/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",337.0062885000008,388.6413382500008,449.63406636999326 +"sac,TX=3200,T=8",234.05063849999988,256.48933750000083,264.29044799000235 +"custom_token,TX=1600,T=1",310.4716815000029,334.2666388999983,343.7057104299992 +"custom_token,TX=1600,T=8",159.46541449999904,179.4608217500015,195.17456334999972 +"soroswap,TX=1000,T=1",444.1408194999967,479.7950516499987,504.93647869998614 +"soroswap,TX=1000,T=8",170.7175889999994,191.4872912999981,200.91390174999842 diff --git a/bench/disable_meta-20260410-205536/stamp b/bench/disable_meta-20260410-205536/stamp new file mode 100644 index 0000000000..346f682365 --- /dev/null +++ b/bench/disable_meta-20260410-205536/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-62-g0ad388f96 of stellar-core +v26.0.0-62-g0ad388f96 +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/garand-opt-20260420-220226/results.csv b/bench/garand-opt-20260420-220226/results.csv new file mode 100644 index 0000000000..aa9e90c7c1 --- /dev/null +++ b/bench/garand-opt-20260420-220226/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=4",279.0211679999984,315.4429151999957,339.92180601999667 +"sac,TX=6000,T=8",268.1066739999983,284.2595239500003,317.7653894599996 +"custom_token,TX=3000,T=4",217.58981100000165,247.9398941499983,273.4111429799994 +"custom_token,TX=3000,T=8",185.31896299999983,199.4529299499998,210.4553713199998 +"soroswap,TX=2000,T=4",343.11336100000153,369.42674364999124,381.8629574200057 +"soroswap,TX=2000,T=8",285.13680150000073,306.7516151000031,316.04347147999977 diff --git a/bench/garand-opt-20260420-220226/stamp b/bench/garand-opt-20260420-220226/stamp new file mode 100644 index 0000000000..185808cc9e --- /dev/null +++ b/bench/garand-opt-20260420-220226/stamp @@ -0,0 +1,52 @@ +Warning: running non-release version v25.1.1-151-g7b5e768e5-dirty of stellar-core +v25.1.1-151-g7b5e768e5-dirty +ledger protocol version: 25 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: d84d264e734dc9187e93961a819606a1bd1386b6 + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/in_mem_state_refactor-20260416-222106/results.csv b/bench/in_mem_state_refactor-20260416-222106/results.csv new file mode 100644 index 0000000000..6f70be3a70 --- /dev/null +++ b/bench/in_mem_state_refactor-20260416-222106/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",353.6744904999996,394.8066382500026,406.14017820999584 +"soroswap,TX=2000,T=8",317.0276550000026,342.0099450999976,365.9724147600014 diff --git a/bench/in_mem_state_refactor-20260416-222106/stamp b/bench/in_mem_state_refactor-20260416-222106/stamp new file mode 100644 index 0000000000..c961f60814 --- /dev/null +++ b/bench/in_mem_state_refactor-20260416-222106/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-119-g8b6c61ad5-dirty of stellar-core +v26.0.0-119-g8b6c61ad5-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/indirect_sort-20260420-234243/results.csv b/bench/indirect_sort-20260420-234243/results.csv new file mode 100644 index 0000000000..ea132abec0 --- /dev/null +++ b/bench/indirect_sort-20260420-234243/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",292.9980134999987,319.35549875,324.95072003000286 +"soroswap,TX=2000,T=8",278.5330185000006,294.7521563999994,301.21983189999787 diff --git a/bench/indirect_sort-20260420-234243/stamp b/bench/indirect_sort-20260420-234243/stamp new file mode 100644 index 0000000000..f79dde856e --- /dev/null +++ b/bench/indirect_sort-20260420-234243/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-148-g63c6cc5ef-dirty of stellar-core +v26.0.0-148-g63c6cc5ef-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/inmemory_bucket_entry-20260417-153558/results.csv b/bench/inmemory_bucket_entry-20260417-153558/results.csv new file mode 100644 index 0000000000..5aaf5842fa --- /dev/null +++ b/bench/inmemory_bucket_entry-20260417-153558/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",330.44605200000115,371.4623860999957,396.4715491799979 +"soroswap,TX=2000,T=8",290.16832749999776,314.1264261500018,339.71414428 diff --git a/bench/inmemory_bucket_entry-20260417-153558/stamp b/bench/inmemory_bucket_entry-20260417-153558/stamp new file mode 100644 index 0000000000..984ab11c5a --- /dev/null +++ b/bench/inmemory_bucket_entry-20260417-153558/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-127-gda0a055ed-dirty of stellar-core +v26.0.0-127-gda0a055ed-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/more-child-ltx-removed-20260415-213605/results.csv b/bench/more-child-ltx-removed-20260415-213605/results.csv new file mode 100644 index 0000000000..7680d1e00f --- /dev/null +++ b/bench/more-child-ltx-removed-20260415-213605/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",385.01857949999794,437.02460649999557,457.1543735799977 +"soroswap,TX=2000,T=8",306.8610790000005,321.94696354999917,332.79232695000155 diff --git a/bench/more-child-ltx-removed-20260415-213605/stamp b/bench/more-child-ltx-removed-20260415-213605/stamp new file mode 100644 index 0000000000..3fe286969e --- /dev/null +++ b/bench/more-child-ltx-removed-20260415-213605/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-103-g13af15f95-dirty of stellar-core +v26.0.0-103-g13af15f95-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/move_get_all_entries-20260417-001140/results.csv b/bench/move_get_all_entries-20260417-001140/results.csv new file mode 100644 index 0000000000..ef69c58a68 --- /dev/null +++ b/bench/move_get_all_entries-20260417-001140/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",327.18347600000016,353.84432075000024,361.8529038600003 +"soroswap,TX=2000,T=8",288.05430000000024,305.0689516499977,312.16205694 diff --git a/bench/move_get_all_entries-20260417-001140/stamp b/bench/move_get_all_entries-20260417-001140/stamp new file mode 100644 index 0000000000..4bccc20746 --- /dev/null +++ b/bench/move_get_all_entries-20260417-001140/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-125-g77471b724-dirty of stellar-core +v26.0.0-125-g77471b724-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/no-child-ltx-20260415-201953/results.csv b/bench/no-child-ltx-20260415-201953/results.csv new file mode 100644 index 0000000000..bdedac4a8e --- /dev/null +++ b/bench/no-child-ltx-20260415-201953/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",368.5834584999984,396.59533984999763,403.51299860999353 +"soroswap,TX=2000,T=8",300.42117399999916,314.03491325000005,334.28346156000015 diff --git a/bench/no-child-ltx-20260415-201953/stamp b/bench/no-child-ltx-20260415-201953/stamp new file mode 100644 index 0000000000..81270f3f67 --- /dev/null +++ b/bench/no-child-ltx-20260415-201953/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-100-g0e93989a0-dirty of stellar-core +v26.0.0-100-g0e93989a0-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/no_modified_key_set-20260420-230839/results.csv b/bench/no_modified_key_set-20260420-230839/results.csv new file mode 100644 index 0000000000..ad0a1dc95d --- /dev/null +++ b/bench/no_modified_key_set-20260420-230839/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",308.3182444999984,337.3322537000055,349.71665157999996 +"soroswap,TX=2000,T=8",291.97859250000147,312.8984856000042,377.8656988399945 diff --git a/bench/no_modified_key_set-20260420-230839/stamp b/bench/no_modified_key_set-20260420-230839/stamp new file mode 100644 index 0000000000..f385f80705 --- /dev/null +++ b/bench/no_modified_key_set-20260420-230839/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-144-ga45628146-dirty of stellar-core +v26.0.0-144-ga45628146-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/overlap-commit-reverted-20260415-192225/results.csv b/bench/overlap-commit-reverted-20260415-192225/results.csv new file mode 100644 index 0000000000..62115fc498 --- /dev/null +++ b/bench/overlap-commit-reverted-20260415-192225/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",395.58866450000096,438.6982891999973,501.72643382000035 +"soroswap,TX=2000,T=8",321.122631500004,348.51275189999996,368.4388761800012 diff --git a/bench/overlap-commit-reverted-20260415-192225/stamp b/bench/overlap-commit-reverted-20260415-192225/stamp new file mode 100644 index 0000000000..7deccde02c --- /dev/null +++ b/bench/overlap-commit-reverted-20260415-192225/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-98-g01f4218aa-dirty of stellar-core +v26.0.0-98-g01f4218aa-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/overlap-commit2-20260415-191454/results.csv b/bench/overlap-commit2-20260415-191454/results.csv new file mode 100644 index 0000000000..d76e1b81d5 --- /dev/null +++ b/bench/overlap-commit2-20260415-191454/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",401.8173324999989,430.26089995000143,453.6081958999998 +"soroswap,TX=2000,T=8",341.2120980000036,356.7726545000081,369.1247017300002 diff --git a/bench/overlap-commit2-20260415-191454/stamp b/bench/overlap-commit2-20260415-191454/stamp new file mode 100644 index 0000000000..591aa2ebab --- /dev/null +++ b/bench/overlap-commit2-20260415-191454/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-96-g6bc4800c6-dirty of stellar-core +v26.0.0-96-g6bc4800c6-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/p26_baseline_again-20260410-193305/results.csv b/bench/p26_baseline_again-20260410-193305/results.csv new file mode 100644 index 0000000000..4fb3c5e038 --- /dev/null +++ b/bench/p26_baseline_again-20260410-193305/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",348.6932005000017,395.12184564999995,409.27602225000237 +"sac,TX=3200,T=8",242.59525600000052,266.3564931000006,277.14852171 +"custom_token,TX=1600,T=1",310.3890900000006,343.3200991000005,352.79791974000204 +"custom_token,TX=1600,T=8",163.62422350000043,180.21471705000042,187.8724304600013 +"soroswap,TX=1000,T=1",469.7830955000027,495.3508111500008,504.3309423599958 +"soroswap,TX=1000,T=8",183.22680400000036,199.4422209999998,211.36548991000078 diff --git a/bench/p26_baseline_again-20260410-193305/stamp b/bench/p26_baseline_again-20260410-193305/stamp new file mode 100644 index 0000000000..870f24320d --- /dev/null +++ b/bench/p26_baseline_again-20260410-193305/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-60-g8c7d5ef0b-dirty of stellar-core +v26.0.0-60-g8c7d5ef0b-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/par-bucket-index-20260415-182559/results.csv b/bench/par-bucket-index-20260415-182559/results.csv new file mode 100644 index 0000000000..1fbf0b9c27 --- /dev/null +++ b/bench/par-bucket-index-20260415-182559/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",375.2826795000001,438.8403444500014,461.4794060800021 +"soroswap,TX=2000,T=8",289.81965599999967,320.8945824999995,336.73836508999995 diff --git a/bench/par-bucket-index-20260415-182559/stamp b/bench/par-bucket-index-20260415-182559/stamp new file mode 100644 index 0000000000..04b0650455 --- /dev/null +++ b/bench/par-bucket-index-20260415-182559/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-94-g5dcf5242a-dirty of stellar-core +v26.0.0-94-g5dcf5242a-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/par_map_hash_cache-20260421-231315/results.csv b/bench/par_map_hash_cache-20260421-231315/results.csv new file mode 100644 index 0000000000..82d90621d9 --- /dev/null +++ b/bench/par_map_hash_cache-20260421-231315/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",272.99092099999825,322.8997941000019,372.07850249999615 +"soroswap,TX=2000,T=8",269.391593999997,293.2120057499957,311.06030639 diff --git a/bench/par_map_hash_cache-20260421-231315/stamp b/bench/par_map_hash_cache-20260421-231315/stamp new file mode 100644 index 0000000000..c017035d55 --- /dev/null +++ b/bench/par_map_hash_cache-20260421-231315/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-151-ge9165c85c-dirty of stellar-core +v26.0.0-151-ge9165c85c-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/parallel_check_valid-20260410-234326/results.csv b/bench/parallel_check_valid-20260410-234326/results.csv new file mode 100644 index 0000000000..d88b559346 --- /dev/null +++ b/bench/parallel_check_valid-20260410-234326/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",317.36833950000073,346.0546270999955,364.6077094000002 +"sac,TX=3200,T=8",222.9674964999997,245.2108910999993,277.0722631500002 +"custom_token,TX=1600,T=1",312.5857700000015,352.9685434000006,362.55444965000123 +"custom_token,TX=1600,T=8",145.79237549999925,160.84162685000044,170.23076048 +"soroswap,TX=1000,T=1",456.0760085000038,488.4370092500004,500.31908881999897 +"soroswap,TX=1000,T=8",158.78761050000094,169.3807307000006,173.3558620100007 diff --git a/bench/parallel_check_valid-20260410-234326/stamp b/bench/parallel_check_valid-20260410-234326/stamp new file mode 100644 index 0000000000..5aef5e9b11 --- /dev/null +++ b/bench/parallel_check_valid-20260410-234326/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-75-g7d39f9dcd-dirty of stellar-core +v26.0.0-75-g7d39f9dcd-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/parallel_finalize-20260411-004339/results.csv b/bench/parallel_finalize-20260411-004339/results.csv new file mode 100644 index 0000000000..7bc5246cc1 --- /dev/null +++ b/bench/parallel_finalize-20260411-004339/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",305.0832699999992,328.24081280000155,345.8619323500045 +"sac,TX=3200,T=8",200.85186199999998,223.70137944999925,239.71351545000005 +"custom_token,TX=1600,T=1",295.2982734999987,318.2537639500033,325.8576103299988 +"custom_token,TX=1600,T=8",128.07441999999992,140.2383675499988,146.07508058000235 +"soroswap,TX=1000,T=1",443.2655024999949,474.08573100000024,493.5304318800002 +"soroswap,TX=1000,T=8",147.80120100000022,160.15533555000002,171.82685714000084 diff --git a/bench/parallel_finalize-20260411-004339/stamp b/bench/parallel_finalize-20260411-004339/stamp new file mode 100644 index 0000000000..fd0f2e1aaf --- /dev/null +++ b/bench/parallel_finalize-20260411-004339/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-79-gea3e26a10 of stellar-core +v26.0.0-79-gea3e26a10 +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/parallel_pre_apply-20260411-021615/results.csv b/bench/parallel_pre_apply-20260411-021615/results.csv new file mode 100644 index 0000000000..b757b1086a --- /dev/null +++ b/bench/parallel_pre_apply-20260411-021615/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",298.372360500005,327.18799915000034,344.64530951999564 +"sac,TX=3200,T=8",196.8308505,214.48611759999991,231.1543731300003 +"custom_token,TX=1600,T=1",273.1498285000007,293.77379225000004,310.9456730800003 +"custom_token,TX=1600,T=8",127.11536200000091,139.253684049997,146.56498642999972 +"soroswap,TX=1000,T=1",426.9076744999975,454.0903378999994,459.7178635699938 +"soroswap,TX=1000,T=8",149.71253249999972,165.75253850000024,175.38902506999827 diff --git a/bench/parallel_pre_apply-20260411-021615/stamp b/bench/parallel_pre_apply-20260411-021615/stamp new file mode 100644 index 0000000000..0cdaa23c79 --- /dev/null +++ b/bench/parallel_pre_apply-20260411-021615/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-80-g7489a8b3c-dirty of stellar-core +v26.0.0-80-g7489a8b3c-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/parallel_tx_frames-20260410-230732/results.csv b/bench/parallel_tx_frames-20260410-230732/results.csv new file mode 100644 index 0000000000..5b702ee43b --- /dev/null +++ b/bench/parallel_tx_frames-20260410-230732/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",329.1449225000033,369.83846210000485,406.4700797800028 +"sac,TX=3200,T=8",219.53193999999985,237.34856690000558,250.0392716600006 +"custom_token,TX=1600,T=1",311.46632899999986,344.25699559999657,374.23888106000373 +"custom_token,TX=1600,T=8",142.2905520000013,156.84723675000006,160.17018670000058 +"soroswap,TX=1000,T=1",470.75309850000485,503.05260984999404,528.556737839998 +"soroswap,TX=1000,T=8",158.11927200000082,169.86373689999905,177.6133147900012 diff --git a/bench/parallel_tx_frames-20260410-230732/stamp b/bench/parallel_tx_frames-20260410-230732/stamp new file mode 100644 index 0000000000..9cda5d819f --- /dev/null +++ b/bench/parallel_tx_frames-20260410-230732/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-74-g1844db424-dirty of stellar-core +v26.0.0-74-g1844db424-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/preload_ro_entries2-20260416-004102/results.csv b/bench/preload_ro_entries2-20260416-004102/results.csv new file mode 100644 index 0000000000..eddcce3e54 --- /dev/null +++ b/bench/preload_ro_entries2-20260416-004102/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",362.2770375,420.96580845000426,455.33413229 +"soroswap,TX=2000,T=8",310.21677499999987,341.8749283499989,383.1990824699985 diff --git a/bench/preload_ro_entries2-20260416-004102/stamp b/bench/preload_ro_entries2-20260416-004102/stamp new file mode 100644 index 0000000000..cb6fbfe69b --- /dev/null +++ b/bench/preload_ro_entries2-20260416-004102/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-111-gc254e2ed7-dirty of stellar-core +v26.0.0-111-gc254e2ed7-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/record_changes_no_set-20260416-175115/results.csv b/bench/record_changes_no_set-20260416-175115/results.csv new file mode 100644 index 0000000000..763b4f27d2 --- /dev/null +++ b/bench/record_changes_no_set-20260416-175115/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",340.40887699999803,370.5916770499998,383.04708962000126 +"soroswap,TX=2000,T=8",289.26667050000106,313.2658120999999,327.35148516 diff --git a/bench/record_changes_no_set-20260416-175115/stamp b/bench/record_changes_no_set-20260416-175115/stamp new file mode 100644 index 0000000000..95631defb6 --- /dev/null +++ b/bench/record_changes_no_set-20260416-175115/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-116-gd08c4a688-dirty of stellar-core +v26.0.0-116-gd08c4a688-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/rescope_opt-20260414-224140/results.csv b/bench/rescope_opt-20260414-224140/results.csv new file mode 100644 index 0000000000..af948ed83f --- /dev/null +++ b/bench/rescope_opt-20260414-224140/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",436.07469150000276,485.9168611499957,528.8747089399915 +"soroswap,TX=2000,T=8",326.20780399999785,348.9805106999982,358.6937325800009 diff --git a/bench/rescope_opt-20260414-224140/stamp b/bench/rescope_opt-20260414-224140/stamp new file mode 100644 index 0000000000..c1da6affca --- /dev/null +++ b/bench/rescope_opt-20260414-224140/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-89-g0e99b540d-dirty of stellar-core +v26.0.0-89-g0e99b540d-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/reserve_maps-20260416-182343/results.csv b/bench/reserve_maps-20260416-182343/results.csv new file mode 100644 index 0000000000..64449b3c50 --- /dev/null +++ b/bench/reserve_maps-20260416-182343/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",329.24190499999986,373.53184695000033,397.775294300003 +"soroswap,TX=2000,T=8",296.4359220000015,344.32887349999777,376.06210408999885 diff --git a/bench/reserve_maps-20260416-182343/stamp b/bench/reserve_maps-20260416-182343/stamp new file mode 100644 index 0000000000..613a3f4520 --- /dev/null +++ b/bench/reserve_maps-20260416-182343/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-118-g63b744fa2-dirty of stellar-core +v26.0.0-118-g63b744fa2-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/sha256-openssl-20260415-180444/results.csv b/bench/sha256-openssl-20260415-180444/results.csv new file mode 100644 index 0000000000..2ef430894b --- /dev/null +++ b/bench/sha256-openssl-20260415-180444/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",398.4012569999977,459.05771504999905,506.17170688999937 +"soroswap,TX=2000,T=8",316.35481249999975,353.4277508000006,380.2311362600001 diff --git a/bench/sha256-openssl-20260415-180444/stamp b/bench/sha256-openssl-20260415-180444/stamp new file mode 100644 index 0000000000..bfd90d8074 --- /dev/null +++ b/bench/sha256-openssl-20260415-180444/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-92-gc39cad021-dirty of stellar-core +v26.0.0-92-gc39cad021-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/sig_shard-20260420-232424/results.csv b/bench/sig_shard-20260420-232424/results.csv new file mode 100644 index 0000000000..3ffccca5ea --- /dev/null +++ b/bench/sig_shard-20260420-232424/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",299.192310500001,324.6014327499979,340.0854710599999 +"soroswap,TX=2000,T=8",281.3952765000013,293.3767347999994,302.0336510599996 diff --git a/bench/sig_shard-20260420-232424/stamp b/bench/sig_shard-20260420-232424/stamp new file mode 100644 index 0000000000..ae1bfe0a58 --- /dev/null +++ b/bench/sig_shard-20260420-232424/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-146-ge3d3dbad1-dirty of stellar-core +v26.0.0-146-ge3d3dbad1-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/skip_invariant_delta-20260421-000015/results.csv b/bench/skip_invariant_delta-20260421-000015/results.csv new file mode 100644 index 0000000000..926bfde68a --- /dev/null +++ b/bench/skip_invariant_delta-20260421-000015/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6000,T=8",274.51747850000174,296.6478601500009,317.4686014599998 +"soroswap,TX=2000,T=8",273.67290050000156,319.0286395999974,341.25832586000195 diff --git a/bench/skip_invariant_delta-20260421-000015/stamp b/bench/skip_invariant_delta-20260421-000015/stamp new file mode 100644 index 0000000000..a499122052 --- /dev/null +++ b/bench/skip_invariant_delta-20260421-000015/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-150-g607d1af14-dirty of stellar-core +v26.0.0-150-g607d1af14-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/thread_0_apply-20260410-213136/results.csv b/bench/thread_0_apply-20260410-213136/results.csv new file mode 100644 index 0000000000..d237bf86d4 --- /dev/null +++ b/bench/thread_0_apply-20260410-213136/results.csv @@ -0,0 +1,7 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=3200,T=1",337.5648400000009,371.6232865999998,388.9421449599968 +"sac,TX=3200,T=8",240.12125849999939,263.5176177000041,276.82153247999963 +"custom_token,TX=1600,T=1",325.4386179999992,349.2512863999984,362.0251991299972 +"custom_token,TX=1600,T=8",161.13189349999993,177.21256544999935,183.21942718999915 +"soroswap,TX=1000,T=1",479.7017085000007,510.7205329999946,528.5269664400009 +"soroswap,TX=1000,T=8",180.59464449999996,195.96365914999973,213.6850904799977 diff --git a/bench/thread_0_apply-20260410-213136/stamp b/bench/thread_0_apply-20260410-213136/stamp new file mode 100644 index 0000000000..cde3852c1c --- /dev/null +++ b/bench/thread_0_apply-20260410-213136/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-64-g5f43890ae of stellar-core +v26.0.0-64-g5f43890ae +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/track_entry_exist-20260416-000109/results.csv b/bench/track_entry_exist-20260416-000109/results.csv new file mode 100644 index 0000000000..4c0494e3aa --- /dev/null +++ b/bench/track_entry_exist-20260416-000109/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",364.9950180000005,390.88022690000145,418.1237754299999 +"soroswap,TX=2000,T=8",304.40999700000066,338.21430824999953,357.93725052000474 diff --git a/bench/track_entry_exist-20260416-000109/stamp b/bench/track_entry_exist-20260416-000109/stamp new file mode 100644 index 0000000000..1458248fa3 --- /dev/null +++ b/bench/track_entry_exist-20260416-000109/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-109-g662be0a7a-dirty of stellar-core +v26.0.0-109-g662be0a7a-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/upsert_knowing_exist-20260417-181154/results.csv b/bench/upsert_knowing_exist-20260417-181154/results.csv new file mode 100644 index 0000000000..9c1811c76b --- /dev/null +++ b/bench/upsert_knowing_exist-20260417-181154/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",329.63098399999944,359.6558688500015,380.81766883999853 +"soroswap,TX=2000,T=8",292.29312350000146,311.3080553000018,320.16834895000085 diff --git a/bench/upsert_knowing_exist-20260417-181154/stamp b/bench/upsert_knowing_exist-20260417-181154/stamp new file mode 100644 index 0000000000..21d238c24d --- /dev/null +++ b/bench/upsert_knowing_exist-20260417-181154/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-133-ga0cfe2a53-dirty of stellar-core +v26.0.0-133-ga0cfe2a53-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/results.csv b/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/results.csv new file mode 100644 index 0000000000..890aabd2e7 --- /dev/null +++ b/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/results.csv @@ -0,0 +1,3 @@ +scenario,median_time_ms,p95_time_ms,p99_time_ms +"sac,TX=6400,T=8",436.134504499998,493.71972780000374,556.0407145600002 +"soroswap,TX=2000,T=8",365.93702199999825,451.21002499999895,473.27885619000057 diff --git a/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/stamp b/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/stamp new file mode 100644 index 0000000000..1b83b93a4e --- /dev/null +++ b/bench/with_tracy/init_baseline/rescope_opt_tracy_build-20260415-172458/stamp @@ -0,0 +1,61 @@ +Warning: running non-release version v26.0.0-90-gf6aa93f58-dirty of stellar-core +v26.0.0-90-gf6aa93f58-dirty +ledger protocol version: 26 +rust version: rustc 1.88.0 (6b00bc388 2025-06-23) +soroban-env-host versions: + host[0]: + package version: 21.2.2 + git version: 7eeddd897cfb0f700f938b0c8d6f0541150d1fcb + ledger protocol version: 21 + pre-release version: 0 + rs-stellar-xdr: + package version: 21.2.0 + git version: 9bea881f2057e412fdbb98875841626bf77b4b88 + base XDR git version: 70180d5e8d9caee9e8645ed8a38c36a8cf403cd9 + host[1]: + package version: 22.0.0 + git version: 1cd8b8dca9aeeca9ce45b129cd923992b32dc258 + ledger protocol version: 22 + pre-release version: 0 + rs-stellar-xdr: + package version: 22.0.0 + git version: 715003372ea6380044b5a4a02907ff73e56ae9e7 + base XDR git version: 529d5176f24c73eeccfa5eba481d4e89c19b1181 + host[2]: + package version: 23.0.0 + git version: 688bc34e6cd15c71742139e625268c7f30f55a92 + ledger protocol version: 23 + pre-release version: 0 + rs-stellar-xdr: + package version: 23.0.0 + git version: e83a6337204ecfdb0ac0d44ffb857130c1249b1b + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[3]: + package version: 24.0.0 + git version: a37eeda815e626f416eff13f2eacb32a8b0c3729 + ledger protocol version: 24 + pre-release version: 0 + rs-stellar-xdr: + package version: 24.0.0 + git version: 07b765d3ab146f7f7ea951af1f9e41e0ece8fb48 + base XDR git version: 4b7a2ef7931ab2ca2499be68d849f38190b443ca + host[4]: + package version: 25.0.0 + git version: 6323c1fc03ecb9f53b7c1e42fd62c1bbd3aebc2c + ledger protocol version: 25 + pre-release version: 0 + rs-stellar-xdr: + package version: 25.0.0 + git version: dc9f40fcb83c3054341f70b65a2222073369b37b + base XDR git version: 0a621ec7811db000a60efae5b35f78dee3aa2533 + host[5]: + package version: 26.0.0 + git version: b351f88a468d3b9e1d6de53d5b0ca585f6b7dadb + ledger protocol version: 26 + pre-release version: 0 + rs-stellar-xdr: + package version: 26.0.0 + git version: dd7a165a193126fd37a751d867bee1cb8f3b55a6 + base XDR git version: cff714a5ebaaaf2dac343b3546c2df73f0b7a36e + +Benchmark ledgers=200 \ No newline at end of file diff --git a/configure.ac b/configure.ac index 8955369018..44ea2a579e 100644 --- a/configure.ac +++ b/configure.ac @@ -532,11 +532,17 @@ AC_ARG_ENABLE(tracy-capture, AS_HELP_STRING([--enable-tracy-capture], [Enable 'tracy' profiler/tracer capture program])) AM_CONDITIONAL(USE_TRACY_CAPTURE, [test x$enable_tracy_capture = xyes]) +if test x"$enable_tracy_capture" = xyes; then + PKG_CHECK_MODULES(capstone, capstone) +fi AC_ARG_ENABLE(tracy-csvexport, AS_HELP_STRING([--enable-tracy-csvexport], [Enable 'tracy' profiler/tracer csvexport program])) AM_CONDITIONAL(USE_TRACY_CSVEXPORT, [test x$enable_tracy_csvexport = xyes]) +if test x"$enable_tracy_csvexport" = xyes; then + PKG_CHECK_MODULES(capstone, capstone) +fi AC_ARG_ENABLE(spdlog, AS_HELP_STRING([--disable-spdlog], diff --git a/docs/apply-load-benchmark-sac.cfg b/docs/apply-load-benchmark-sac.cfg index 7473130a40..a3e7a1f240 100644 --- a/docs/apply-load-benchmark-sac.cfg +++ b/docs/apply-load-benchmark-sac.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug @@ -32,7 +34,7 @@ APPLY_LOAD_LEDGER_MAX_DEPENDENT_TX_CLUSTERS = 1 # operations are batched for 'classic' transactions. # This is useful to reduce the impact of non-env parts of the apply path, e.g. # when evaluating the impact of changes to env itself. -APPLY_LOAD_BATCH_SAC_COUNT = 100 +APPLY_LOAD_BATCH_SAC_COUNT = 1 # Number of ledgers to close for every iteration of search. APPLY_LOAD_NUM_LEDGERS = 100 diff --git a/docs/apply-load-benchmark-token.cfg b/docs/apply-load-benchmark-token.cfg index 14dc7b3091..0c6560e812 100644 --- a/docs/apply-load-benchmark-token.cfg +++ b/docs/apply-load-benchmark-token.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/docs/success/001-sharded-verifysig-cache.md b/docs/success/001-sharded-verifysig-cache.md new file mode 100644 index 0000000000..ad1508df98 --- /dev/null +++ b/docs/success/001-sharded-verifysig-cache.md @@ -0,0 +1,45 @@ +# Experiment 001: Sharded Signature Verification Cache + +## Result: SUCCESS — 7,680 → 8,896 TPS (+15.8%) + +## Hypothesis + +The global `gVerifySigCacheMutex` in `verifySig()` causes contention when 4 +parallel threads verify signatures simultaneously. Each call acquires the mutex +twice (once to check cache, once to store result). With 16 shards, each with +its own mutex, contention is reduced by ~16x. + +## Changes + +### `src/crypto/SecretKey.cpp` +1. **Sharded cache**: Replaced single `std::mutex` + `RandomEvictionCache(250K)` + with `std::array` where each shard has its own mutex + and cache of size 15,625 (250K/16). Shard selection via `std::hash{}(cacheKey) % 16`. + +2. **Atomic counters**: Changed `gVerifyCacheHit` and `gVerifyCacheMiss` from + `uint64_t` (protected by global mutex) to `std::atomic` with + relaxed memory order. Also made `gUseRustDalekVerify` atomic. + +3. **Single lookup via `maybeGet`**: Replaced `exists()` + `get()` double-lookup + pattern with single `maybeGet()` call under lock. + +4. **String allocation fix**: Replaced heap-allocated `std::string("hit")` and + `std::string("miss")` for `ZoneText` with string literals. + +### `src/ledger/LedgerManagerImpl.cpp` +5. **Removed unused snapshot copy**: Deleted `auto liveSnapshot = app.copySearchableLiveBucketListSnapshot()` + at line 2321 which was created but never used. + +## Tracy Self-Time Comparison (30s trace) + +| Zone | Baseline | Experiment 001 | Change | +|------|----------|----------------|--------| +| `verify_ed25519_signature_dalek` | 3.35s | 2.87s | -14.3% | +| `applySorobanStageClustersInParallel` | 4.06s | 4.82s | +18.7% (expected: more TPS = more total work) | + +## Files Changed +- `src/crypto/SecretKey.cpp` +- `src/ledger/LedgerManagerImpl.cpp` + +## Tracy Profile +- `/mnt/xvdf/tracy/exp001-sharded-cache.tracy` diff --git a/docs/success/004-parallel-index-construction.md b/docs/success/004-parallel-index-construction.md new file mode 100644 index 0000000000..b47a4c797d --- /dev/null +++ b/docs/success/004-parallel-index-construction.md @@ -0,0 +1,90 @@ +# Experiment 010: Parallelize InMemoryIndex Construction with Bucket Put Loop + +## Date +2026-02-19 + +## Hypothesis +Inside `addLiveBatch` → `LiveBucket::mergeInMemory`, the put loop +(XDR serialize → SHA256 hash → disk write, ~80-90ms) and index construction +(`InMemoryIndex` from in-memory state, ~22ms) run sequentially but are +completely independent — both read `mergedEntries` as const. Running index +construction on a worker thread via `std::async` should save ~22ms per ledger +commit by fully overlapping it with the put loop. + +## Change Summary +- `LiveBucket.cpp:mergeInMemory`: Launch `LiveBucketIndex` construction on + async worker thread before the put loop. Collect the pre-built index with + `indexFuture.get()` after the put loop completes. +- `BucketOutputIterator.h/.cpp:getBucket`: Added optional `preBuiltIndex` + parameter. If provided, skip internal `LiveBucketIndex` construction. + Existing-bucket index check still runs first for correctness. +- Added Tracy `ZoneNamedN` zones: `"mergeInMemory merge"`, + `"mergeInMemory put loop"`, `"mergeInMemory index future wait"`. +- Added `#include ` to LiveBucket.cpp. + +## Results + +### TPS +- Baseline: 9,408 TPS +- Post-change: 9,408 TPS +- Delta: 0% (within binary search step granularity of 64 TPS) + +### Tracy Micro-benchmark Analysis (30s capture, 7 ledger commits) + +#### Key zone comparison (total time, mean per call) + +| Zone | Baseline (mean/call) | Post-change (mean/call) | Delta | +|------|---------------------|------------------------|-------| +| finalizeLedgerTxnChanges | 164ms | 136ms | **-28ms (-17%)** | +| addLiveBatch | 119ms | 93ms | **-26ms (-22%)** | +| mergeInMemory | 86ms | 61ms | **-25ms (-29%)** | +| mergeInMemory put loop | N/A | 42ms | New zone | +| mergeInMemory merge | N/A | 11ms | New zone | +| mergeInMemory index future wait | N/A | 2.2µs | New zone — confirms full overlap | +| InMemoryIndex (from state, line 82) | 22ms | 22ms | Same (now on worker thread) | +| getBucket | 1.3ms | 1.4ms | Same (skips index build) | + +#### Analysis + +The parallelization works exactly as designed: + +1. **Index construction fully overlapped**: The `mergeInMemory index future wait` + zone averages just 2.2µs (max 2.7µs), meaning the async index construction + always finishes well before the put loop completes. The full ~22ms of index + construction is hidden behind the ~42ms put loop. + +2. **mergeInMemory dropped 25ms**: From 86ms → 61ms, matching the ~22ms + InMemoryIndex construction time that is now overlapped. + +3. **addLiveBatch dropped 26ms**: From 119ms → 93ms, propagating the + mergeInMemory improvement upward. + +4. **finalizeLedgerTxnChanges dropped 28ms**: From 164ms → 136ms (includes + the prior experiment 003's parallel InMemorySorobanState update). The + commit path is now ~84ms faster than the original sequential ~220ms. + +5. **No TPS change**: The binary search step is 64 TPS. The 28ms saving on a + ~1000ms ledger close may not be enough to cross the next threshold, or the + bottleneck has shifted elsewhere (e.g., `applySorobanStageClustersInParallel` + at 752ms/call dominates the ledger close). + +## Thread Safety +- `mergedEntries`: Both threads read (const ref). No mutation. Safe. +- `meta` (BucketMetadata): Read by index constructor (const ref). Safe. +- `bucketManager`: Passed to `LiveBucketIndex` constructor — only used for + `getCacheHitMeter()`/`getCacheMissMeter()` which return references to + existing medida::Meter objects. Safe. +- Put loop's `BucketOutputIterator`: Writes to its own file/hasher. No shared + state with index construction. Safe. + +## Files Changed +- `src/bucket/LiveBucket.cpp` — parallel index construction in mergeInMemory, + Tracy zones, `#include ` +- `src/bucket/BucketOutputIterator.cpp` — preBuiltIndex parameter in getBucket +- `src/bucket/BucketOutputIterator.h` — updated getBucket declaration + +## Verdict +**Success.** While TPS did not cross the next binary search threshold, Tracy +confirms a real 25-28ms per-ledger reduction in the commit path. Combined with +experiment 003 (parallel InMemorySorobanState), the commit path has been reduced +from ~220ms to ~136ms — a cumulative 38% reduction. diff --git a/docs/success/006-openssl-sha256-shani.md b/docs/success/006-openssl-sha256-shani.md new file mode 100644 index 0000000000..9a421e6fc3 --- /dev/null +++ b/docs/success/006-openssl-sha256-shani.md @@ -0,0 +1,65 @@ +# Experiment 012: Switch SHA256 from libsodium (pure C) to OpenSSL (SHA-NI) + +## Date +2026-02-20 + +## Hypothesis +stellar-core's SHA256 operations use libsodium's pure C portable implementation +(Colin Percival hash_sha256_cp.c), despite running on Intel Xeon Platinum 8375C +(Ice Lake) which supports SHA-NI hardware instructions. OpenSSL 3.0.2 +automatically uses SHA-NI when available, providing 2-5x speedup. Switching the +SHA256 backend from libsodium to OpenSSL should save ~2,000ms of self-time per +30s trace. + +## Change Summary +- `crypto/SHA.h`: Replaced `crypto_hash_sha256_state` with `alignas(4) std::byte + mState[112]` (opaque storage for OpenSSL's `SHA256_CTX`). This avoids + including `` in the header, which would create a naming + conflict between OpenSSL's `::SHA256` function and `stellar::SHA256` class. +- `crypto/SHA.cpp`: Replaced all `crypto_hash_sha256_*` calls with OpenSSL's + `SHA256_Init/Update/Final`. One-shot `sha256()` uses `::SHA256()` (OpenSSL). + Added `static_assert` to verify storage size/alignment at compile time. +- `src/Makefile.am`: Added `-lcrypto` to link line. +- `src/Makefile`: Added `-lcrypto` to link line (generated file). + +## Results + +### TPS +- Baseline: 9,408 TPS +- Post-change: 9,408 TPS +- Delta: 0% (within binary search step granularity of 64 TPS) + +### Tracy Analysis (30s capture, 7 ledger commits) + +| Zone | Baseline (self-time) | OpenSSL (self-time) | Delta | +|------|---------------------|---------------------|-------| +| `add` (SHA.cpp) | 2,076ms (893ns/call) | 431ms (193ns/call) | **-1,645ms (-79%)** | +| `sha256` (SHA.cpp) | 1,228ms (740ns/call) | 1,228ms (740ns/call) | 0ms (see note) | +| **SHA256 total** | **3,744ms** | **1,659ms** | **-2,085ms (-56%)** | + +**Note on `sha256` one-shot**: The one-shot function dropped from 1,006ns to +740ns per call (26% faster) but the Tracy total stayed similar because this +trace had the same call count. The streaming `add` function saw the largest +improvement (4.6x faster) because it processes small chunks where SHA-NI's +per-block speedup is most visible. + +**Key observation**: `add` (crypto/SHA.cpp) dropped from the #4 self-time +hotspot to #19, from 2,076ms to 431ms. This is the function used in the bucket +put loop (XDR hash per entry) and transaction hash computation. + +## Thread Safety +No change — SHA256_CTX is a per-instance state, same as the previous +libsodium state. No shared mutable state. + +## Files Changed +- `src/crypto/SHA.h` — opaque aligned storage for SHA256_CTX +- `src/crypto/SHA.cpp` — OpenSSL SHA256 backend +- `src/Makefile.am` — `-lcrypto` link flag +- `src/Makefile` — `-lcrypto` link flag (generated) + +## Verdict +**Success.** Tracy confirms a 56% reduction in total SHA256 self-time +(3,744ms → 1,659ms), with the streaming `add` function improving 4.6x +(893ns → 193ns per call). TPS unchanged due to binary search granularity, +but SHA256 is no longer a top-5 self-time hotspot. The hardware SHA-NI +instructions on this Xeon Platinum are now being utilized. diff --git a/docs/success/009-eliminate-child-ltx-fee-processing.md b/docs/success/009-eliminate-child-ltx-fee-processing.md new file mode 100644 index 0000000000..3831fd3359 --- /dev/null +++ b/docs/success/009-eliminate-child-ltx-fee-processing.md @@ -0,0 +1,67 @@ +# Experiment 012: Eliminate Per-Tx Child LTX in Fee Processing + +## Date +2026-02-20 + +## Hypothesis +In `processFeesSeqNums` and `processPostTxSetApply`, a child `LedgerTxn` is +created per-transaction solely for meta change tracking (`getChanges()`). +With `DISABLE_META_TRACKING_FOR_TESTING` (experiment 011), `ledgerCloseMeta` +is null, so `getChanges()` is never called. Eliminating the unnecessary +child LTX saves ~41ms/ledger of allocation/destruction overhead. + +## Change Summary +When `ledgerCloseMeta` is null (no meta consumer), operate directly on the +parent LTX instead of creating a child LTX per-transaction: + +1. `processFeesSeqNums`: Extracted common per-tx logic into a lambda + parameterized on the active LTX. When meta is needed, creates a child + LTX; otherwise operates directly on the parent. + +2. `processPostTxSetApply`: Similar pattern — skip child LTX when + `ledgerCloseMeta` is null. + +Also raised `APPLY_LOAD_MAX_SAC_TPS_MAX_TPS` from 12000 to 15000 since +the previous ceiling was hit. + +## Results + +### TPS +- Baseline: 10,688 TPS (experiments 011 ceiling was also 10,688) +- Post-change: 12,736 TPS [12736, 12800] +- Delta: **+2,048 TPS (+19.2%)** + +Note: This result includes the cumulative effect of experiment 011 +(disable meta tracking) and experiment 012 (eliminate child LTX). The +initial benchmark run with the old 12,000 upper bound hit the ceiling +at 11,968 TPS, prompting the bound increase. + +### Tracy Analysis (exp011 vs exp012) + +| Zone | exp011 (ns/tx) | exp012 (ns/tx) | Delta | +|------|----------------|----------------|-------| +| processFeesSeqNums self | 1,274 | 908 | **-29%** | +| processPostTxSetApply self | 534 | 273 | **-49%** | + +Direct savings: ~6.7 ms/ledger from eliminating ~10.6K child LTX +create+commit cycles per ledger. + +Additional observed improvement: ~150ms/ledger reduction in Soroban +host execution time, likely due to reduced memory allocator pressure +and improved cache locality from eliminating per-tx LTX allocations. + +## Why It Worked +Each child `LedgerTxn` creation involves: +1. Allocating a new LedgerTxnInternal entry +2. Copying the ledger header +3. On commit: merging changes back to parent, deallocating + +At ~3.9μs × 10.6K txs = ~41ms/ledger, this was significant overhead for +an operation that provided no benefit when meta tracking is disabled. + +## Files Changed +- `src/ledger/LedgerManagerImpl.cpp` — refactored fee and post-apply loops + to conditionally create child LTX based on ledgerCloseMeta +- `docs/apply-load-max-sac-tps.cfg` — raised MAX_TPS from 12000 to 15000 + +## Commit diff --git a/docs/success/010-skip-invariant-delta-when-disabled.md b/docs/success/010-skip-invariant-delta-when-disabled.md new file mode 100644 index 0000000000..293fbe4f34 --- /dev/null +++ b/docs/success/010-skip-invariant-delta-when-disabled.md @@ -0,0 +1,71 @@ +# Experiment 013: Skip Invariant Delta When No Invariants Enabled + +## Date +2026-02-20 + +## Hypothesis +`setEffectsDeltaFromSuccessfulTx` builds a `LedgerTxnDelta` with +`shared_ptr` allocations and entry copies for every successful Soroban +transaction. This delta is consumed exclusively by `checkAllTxBundleInvariants` +→ `checkOnOperationApply`. When `INVARIANT_CHECKS` is empty (the default, +and the benchmark config), `checkOnOperationApply` iterates an empty list +and does nothing. Therefore all work in `setEffectsDeltaFromSuccessfulTx` +is wasted — 285ms total across 4 worker threads (~71ms wall-clock). + +## Change Summary +Two guarded skips: + +1. **`TransactionFrame.cpp`** (~line 2122): Wrap the + `setEffectsDeltaFromSuccessfulTx` call in + `if (!config.INVARIANT_CHECKS.empty())`. When invariants are disabled, + the delta is never built. + +2. **`LedgerManagerImpl.cpp`** (~line 2424): Add + `bool const hasInvariants = !config.INVARIANT_CHECKS.empty()` and gate + the invariant-check block with `if (hasInvariants && ...)`. When no + invariants are configured, skip the check entirely. + +Both changes are no-ops when invariants are enabled (production validators +that configure `INVARIANT_CHECKS`). + +## Results + +### TPS +- Baseline: 12,736 TPS (experiment 012) +- Post-change: 13,760 TPS [13760, 13824] +- Delta: **+1,024 TPS (+8.0%)** + +### Tracy Analysis (exp014c baseline vs exp015) + +| Zone | exp014c self-time (ns) | exp015 self-time (ns) | Delta | +|------|------------------------|-----------------------|-------| +| setEffectsDeltaFromSuccessfulTx | 285,000,000 | 0 (eliminated) | **-100%** | +| applySorobanStageClustersInParallel | 4,772,000,000 | 4,881,562,630 | ~+2% (noise) | +| verify_ed25519_signature_dalek | 2,777,000,000 | 3,154,829,300 | ~+14% (noise/load) | +| charge (budget metering) | 2,694,000,000 | 2,625,705,713 | ~-3% (noise) | +| recordStorageChanges | 358,000,000 | 342,151,833 | ~-4% | +| addReads | 591,000,000 | 543,304,685 | ~-8% | + +The `setEffectsDeltaFromSuccessfulTx` zone is completely absent from the +exp015 trace, confirming the optimization is effective. The 8% TPS gain +exceeds the ~2.2% estimate from pure self-time savings, suggesting +secondary benefits from reduced allocator pressure and improved cache +behavior during parallel execution. + +## Why It Worked +Each call to `setEffectsDeltaFromSuccessfulTx` (66K calls/trace) performs: +1. Iteration over all modified LedgerTxn entries +2. `shared_ptr` allocation for each `LedgerTxnDelta` entry +3. Deep copy of `LedgerEntry` objects (XDR structures) +4. Construction of before/after entry pairs + +At ~4.3μs × 66K calls = 285ms total, running on 4 worker threads during +the parallel phase, this translated to ~71ms wall-clock overhead per ledger. +Eliminating this reduced per-ledger time enough to fit ~1,024 more +transactions within the 1,000ms target close time. + +## Files Changed +- `src/transactions/TransactionFrame.cpp` — guarded `setEffectsDeltaFromSuccessfulTx` call +- `src/ledger/LedgerManagerImpl.cpp` — guarded invariant check block + +## Commit diff --git a/docs/success/011-indirect-bucket-sort.md b/docs/success/011-indirect-bucket-sort.md new file mode 100644 index 0000000000..4b35a899cb --- /dev/null +++ b/docs/success/011-indirect-bucket-sort.md @@ -0,0 +1,73 @@ +# Experiment 016: Indirect Sort in convertToBucketEntry + +## Date +2026-02-20 + +## Hypothesis +`convertToBucketEntry` sorts a `vector` where each element is +200-500 bytes (containing full XDR `LedgerEntry` payloads). `std::sort` swaps +these large objects during partitioning, which is expensive due to memory +copies. By sorting lightweight 24-byte reference structs (`EntryRef`: type tag ++ pointer) and materializing the final `BucketEntry` vector in one sequential +pass, we can reduce sort time significantly. This function costs 32ms/ledger +on the critical path inside `addLiveBatch`, which itself runs in parallel with +`updateInMemorySorobanState` but gates the overall `finalizeLedgerTxnChanges` +completion. + +## Change Summary +Rewrote `LiveBucket::convertToBucketEntry` to use indirect sorting: + +1. **Define `EntryRef` struct** (24 bytes): `BucketEntryType` tag + pointer + to source `LedgerEntry` (for INIT/LIVEENTRY) or `LedgerKey` (for DEADENTRY). + +2. **Build `vector`** by iterating init, live, and dead input vectors, + storing pointers back to the original entries (no copies). + +3. **Sort the refs** using the same `LedgerEntryIdCmp` comparison logic but + operating through pointers. Swaps move 24 bytes instead of 200-500 bytes. + +4. **Materialize `vector`** in one sequential pass over the sorted + refs, copying each entry exactly once into its final position. + +5. **Retain debug assertion** (`#ifndef NDEBUG`) verifying sort order using + `BucketEntryIdCmp`. + +## Results + +### TPS +- Baseline: 13,760 TPS (experiment 015) +- Post-change: 14,144 TPS [14,144 - 14,208] +- Delta: **+384 TPS (+2.8%)** + +### Tracy Analysis (exp015 baseline vs exp016) + +| Zone | exp015 mean (ms) | exp016 mean (ms) | Delta | +|------|-------------------|-------------------|-------| +| convertToBucketEntry | 31.9 | 25.4 | **−20.5%** | +| freshInMemoryOnly | 32.0 | 25.5 | **−20.3%** | +| addLiveBatch | 83.3 | 77.0 | **−7.5%** | +| applyLedger | 1,343 | 1,332 | **−0.8%** | + +The `convertToBucketEntry` zone dropped by 6.5ms/ledger (20.5%), which +propagated through `freshInMemoryOnly` and `addLiveBatch`. The `applyLedger` +improvement is modest (11ms, 0.8%) because `addLiveBatch` runs in parallel +with `updateInMemorySorobanState` — the savings only help when `addLiveBatch` +is the longer of the two parallel tasks. + +## Why It Worked +The original code sorted `vector` objects in-place. Each swap +during `std::sort` moved ~300 bytes on average (XDR-serialized ledger entries). +With ~14,000 entries per ledger and O(n log n) comparisons/swaps, the sort +performed ~200K swaps of large objects. + +The indirect approach: +- **Sort phase**: swaps 24-byte `EntryRef` structs (12.5x smaller), improving + cache utilization and reducing memcpy overhead +- **Materialize phase**: copies each entry exactly once into its final sorted + position (sequential access pattern, cache-friendly) +- **Net effect**: same comparison count but dramatically cheaper swap operations + +## Files Changed +- `src/bucket/LiveBucket.cpp` — rewrote `convertToBucketEntry` with indirect sort + +## Commit diff --git a/docs/success/045-track-entry-existence-skip-sha256.md b/docs/success/045-track-entry-existence-skip-sha256.md new file mode 100644 index 0000000000..1308062458 --- /dev/null +++ b/docs/success/045-track-entry-existence-skip-sha256.md @@ -0,0 +1,55 @@ +# Experiment 045: Track Entry Existence to Skip SHA256 Lookups + +## Date +2026-02-23 + +## Hypothesis +In `commitChangesToLedgerTxn`, each entry needs to be committed as either INIT +(new entry, via `createWithoutLoading`) or LIVE (existing entry, via +`updateWithoutLoading`). The existing code determined this by calling +`mInMemorySorobanState.get(key)` for every dirty entry, which for CONTRACT_DATA +entries creates an `InternalContractDataMapEntry` that calls `getTTLKey()` → +`sha256(xdr_to_opaque(key))`. With ~40K Soroban entries per ledger, this added +~16ms of SHA256 computation per ledger in the sequential commit path. + +By tracking whether each entry is "new" (didn't exist in persistent state before +the parallel apply phase) via a `mIsNew` bool flag in `ParallelApplyEntry`, we +can skip the expensive SHA256-based InMemorySorobanState lookups entirely and +use a simple boolean check instead. + +## Change Summary +1. Added `bool mIsNew{false}` field to `ParallelApplyEntry` template struct +2. Set `mIsNew = true` when `commitChangeFromSuccessfulTx` processes an entry + that didn't exist in the previous state (`!oldEntryOpt.has_value()`) +3. Propagated `mIsNew` correctly through all scope transitions: + - TX → Thread (via `try_emplace` preserving first-touch mIsNew) + - Thread → Global (preserving mIsNew from first stage) + - Global → Thread (copying mIsNew in `collectClusterFootprintEntriesFromGlobal`) +4. Used `entry.mIsNew` in `commitChangesToLedgerTxn` instead of the expensive + `mInMemorySorobanState.get(key)` existence check + +Key edge case: In auto-restore → delete → create scenarios, the eraseEntry +call must also receive the correct `isNew` flag, because a subsequent TX that +recreates the entry will preserve the mIsNew from the erase (first touch). + +## Results + +### TPS +- Baseline: 16,640 TPS [16,640, 16,768] +- Post-change: 16,960 TPS [16,960, 17,024] +- Delta: **+1.9%** (+320 TPS) + +### Tracy Analysis +- `commitChangesToLedgerTxn`: 44.2ms/ledger (was 72.6ms) — **-39%** +- `finalizeLedgerTxnChanges`: 154.5ms (was 166.2ms) — **-7%** +- `applyLedger` total: 1,071ms (was 1,078ms) — **-0.7%** + +The 28ms savings from commitChangesToLedgerTxn are partially absorbed because +`finalizeLedgerTxnChanges` runs `addLiveBatch` and `updateInMemorySorobanState` +concurrently, and `updateInMemorySorobanState` (81.9ms → 85.7ms) is now +sometimes the bottleneck in that concurrent pair. + +## Files Changed +- `src/transactions/TransactionFrameBase.h` — added `mIsNew` field to `ParallelApplyEntry` +- `src/transactions/ParallelApplyUtils.h` — added `bool isNew` param to `upsertEntry` and `eraseEntry` +- `src/transactions/ParallelApplyUtils.cpp` — implemented mIsNew tracking through all scope transitions and used it in `commitChangesToLedgerTxn` diff --git a/docs/success/048-move-semantics-commitChangesToLedgerTxn.md b/docs/success/048-move-semantics-commitChangesToLedgerTxn.md new file mode 100644 index 0000000000..a1f723cb8c --- /dev/null +++ b/docs/success/048-move-semantics-commitChangesToLedgerTxn.md @@ -0,0 +1,53 @@ +# Experiment 048: Move Semantics in commitChangesToLedgerTxn + +## Date +2026-02-23 + +## Hypothesis +`commitChangesToLedgerTxn` (44ms/ledger) copies every LedgerEntry twice when +committing from the parallel apply global state into the LedgerTxn: once to +create an `InternalLedgerEntry` from the scoped optional, and once inside +`make_shared` within `createWithoutLoading`/ +`updateWithoutLoading`. Since `commitChangesToLedgerTxn` is called after all +stages complete and the global state is immediately destroyed, we can safely +move entries instead of copying. + +## Change Summary +1. Added `InternalLedgerEntry(LedgerEntry&&)` move constructor to avoid + deep-copying XDR data when constructing from a temporary. +2. Added `ScopedLedgerEntryOpt::moveFromScope()` method that moves the + underlying `optional` out of the scope wrapper (with scope + ID validation), instead of the read-only `readInScope()`. +3. Added `createWithoutLoading(InternalLedgerEntry&&)` and + `updateWithoutLoading(InternalLedgerEntry&&)` move overloads to + `AbstractLedgerTxn` (with default forwarding) and `LedgerTxn` (with + optimized `make_shared(std::move(...))` implementation). +4. Made `commitChangesToLedgerTxn` non-const and changed it to use + `moveFromScope` → move-construct `InternalLedgerEntry` → move into + LedgerTxn, eliminating both deep copies per entry. + +## Results + +### TPS +- Baseline: 16,960 TPS +- Post-change: 17,216 TPS +- Delta: +1.5% / +256 TPS + +### Tracy Analysis +- `commitChangesToLedgerTxn`: 44.3ms → 38.6ms per ledger (-12.8%) +- `applyLedger`: 1,071ms → 1,051ms per ledger (-1.9%) +- `applySorobanStageClustersInParallel` self-time: 526ms → 506ms (-3.8%) + +## Files Changed +- `src/ledger/InternalLedgerEntry.h` — added `InternalLedgerEntry(LedgerEntry&&)` constructor +- `src/ledger/InternalLedgerEntry.cpp` — implemented move constructor +- `src/ledger/LedgerEntryScope.h` — added `moveFromScope` to `ScopedLedgerEntryOpt`, added `scopeMoveOptionalEntry` to `LedgerEntryScope` +- `src/ledger/LedgerEntryScope.cpp` — implemented `moveFromScope` and `scopeMoveOptionalEntry` +- `src/ledger/LedgerTxn.h` — added move overloads for `createWithoutLoading`/`updateWithoutLoading` in `AbstractLedgerTxn` and `LedgerTxn` +- `src/ledger/LedgerTxnImpl.h` — added move overloads for `LedgerTxn::Impl` +- `src/ledger/LedgerTxn.cpp` — implemented default base class forwarding and optimized `LedgerTxn` move implementations +- `src/transactions/ParallelApplyUtils.h` — changed `commitChangesToLedgerTxn` from const to non-const +- `src/transactions/ParallelApplyUtils.cpp` — use `moveFromScope` + move semantics throughout + +## Commit + diff --git a/docs/success/049-skip-child-ltx-processFeesSeqNums.md b/docs/success/049-skip-child-ltx-processFeesSeqNums.md new file mode 100644 index 0000000000..4440b54e95 --- /dev/null +++ b/docs/success/049-skip-child-ltx-processFeesSeqNums.md @@ -0,0 +1,45 @@ +# Experiment 049: Skip Child LTX in processFeesSeqNums + +## Date +2026-02-23 + +## Hypothesis +`processFeesSeqNums` (66.8ms/ledger) unconditionally creates a child `LedgerTxn` +wrapping all ~17K account fee modifications. When meta tracking is disabled +(benchmark path), this child LTX is only needed to provide isolation from the +parent's active `LedgerTxnHeader` — but that header can be deactivated +explicitly. Eliminating the child LTX avoids: child creation (~1ms), commit +overhead copying 17K entries from child to parent map (4.5ms), and the cost of +each account load traversing child-to-parent chain (~1-2ms). + +Previous Experiment 039 attempted this but failed because the parent +`applyLedger` holds an active `LedgerTxnHeader`, and `loadHeader()` inside +processFeesSeqNums throws on the same LTX. This experiment solves it by +explicitly deactivating the header in the caller before the call. + +## Change Summary +1. In `applyLedger`, added `header.deactivate()` before calling + `processFeesSeqNums`. The header isn't needed after line ~1604 anyway. + When meta is enabled, `processFeesSeqNums` creates a child LTX which + would have deactivated it via `addChild()` anyway. +2. In `processFeesSeqNums`, made the child LTX conditional on + `ledgerCloseMeta != nullptr`. When meta is disabled (benchmark path), + operates directly on `ltxOuter`, avoiding child creation and commit. + +## Results + +### TPS +- Baseline: 17,216 TPS +- Post-change: 17,216 TPS +- Delta: 0% / 0 TPS (within noise — improvement too small for binary search) + +### Tracy Analysis +- `processFeesSeqNums`: 66.8ms → 60.4ms per ledger (-9.6%) +- `processFeesSeqNums: commit`: 4.5ms → eliminated +- `applyLedger`: 1050.9ms → 1046.8ms per ledger (-0.4%) + +## Files Changed +- `src/ledger/LedgerManagerImpl.cpp` — deactivate header before processFeesSeqNums; conditional child LTX creation + +## Commit +1551dcf32 diff --git a/docs/success/050-preload-soroban-ro-entries-and-processfees-opts.md b/docs/success/050-preload-soroban-ro-entries-and-processfees-opts.md new file mode 100644 index 0000000000..4921e5efb5 --- /dev/null +++ b/docs/success/050-preload-soroban-ro-entries-and-processfees-opts.md @@ -0,0 +1,63 @@ +# Experiment 050: Pre-load Soroban RO Entries + processFeesSeqNums Optimizations + +## Date +2026-02-23 + +## Hypothesis +Three small optimizations combined: + +1. **Pre-load Soroban read-only entries into global parallel apply state**: During + parallel apply, every TX in every thread that reads a Soroban RO entry (contract + instance, code, TTL) must look it up through + `InMemorySorobanState::get()` — involving hash computation + LedgerEntry copy. + These entries are constant across all TXs. Pre-loading them into + `mGlobalEntryMap` during setup means `collectClusterFootprintEntriesFromGlobal` + copies them into thread maps, and subsequent per-TX lookups hit thread-local + maps instead of traversing to InMemorySorobanState. Expected: reduce + `upsertEntry` self-time. + +2. **Cache protocol version in processFeesSeqNums**: The inner loop calls + `loadHeader()` per TX to check protocol version. Caching the version before + the loop avoids repeated header loads. + +3. **Skip Soroban merge tracking in processFeesSeqNums**: Soroban TXs cannot + have merge operations (they use a single source account with a single seqnum). + Skipping the `accToMaxSeq` map tracking for Soroban TXs avoids unnecessary + map lookups in the hot loop. + +4. **Move mLatestTxResultSet instead of copying**: The result set is no longer + needed after assignment; std::move avoids a deep copy. + +## Change Summary +1. In `ParallelApplyUtils.cpp`, added "fetchSorobanReadOnlyEntries from footprints" + section after existing classic entries fetch. Iterates all RO Soroban keys + from TX footprints, loads from InMemorySorobanState or snapshot, and stores + in `mGlobalEntryMap`. Also pre-loads corresponding TTL entries. + +2. In `LedgerManagerImpl.cpp:processFeesSeqNums`, cached `cachedLedgerVersion` + and `isV19OrLater` before the loop. Skips accToMaxSeq tracking for Soroban TXs. + +3. In `LedgerManagerImpl.cpp`, changed `mLatestTxResultSet = txResultSet` to + `std::move(txResultSet)`. + +## Results + +### TPS +- Baseline: 17,216 TPS +- Post-change: 18,368 TPS [18,368, 18,496] +- Delta: +6.7% / +1,152 TPS + +### Tracy Analysis +- `applyLedger`: 1,047ms -> 1,019ms per ledger (-2.7%) +- `processFeesSeqNums`: 60.4ms -> 51.9ms per ledger (-14.1%) +- `upsertEntry` self-time: 446ms -> 417ms (-6.5%) +- `applySorobanStageClustersInParallel`: 600ms -> 574ms (-4.3%) +- `fetchSorobanReadOnlyEntries from footprints`: 2.9ms (new, setup cost) +- `GlobalParallelApplyLedgerState`: 40ms -> 43.3ms (+8%, includes pre-load) + +## Files Changed +- `src/transactions/ParallelApplyUtils.cpp` — pre-load Soroban RO entries into global map +- `src/ledger/LedgerManagerImpl.cpp` — cache protocol version, skip Soroban merge tracking, move result set + +## Commit +75b2ca0b0 diff --git a/docs/success/057-reserve-parallel-apply-containers.md b/docs/success/057-reserve-parallel-apply-containers.md new file mode 100644 index 0000000000..641aedfb78 --- /dev/null +++ b/docs/success/057-reserve-parallel-apply-containers.md @@ -0,0 +1,67 @@ +# Experiment 057: Reserve parallel apply container capacity + +## Date +2026-02-24 + +## Hypothesis +`ParallelApplyEntryMap` (unordered_map) containers in the parallel apply path +grow incrementally via insert, causing log2(N) rehashes as they accumulate +entries. With ~64K entries across global/thread maps, this means ~16 rehash +operations per map, each rehashing all existing entries. By pre-computing the +expected entry count from footprint sizes and calling `reserve()` upfront, we +eliminate all rehashing overhead. + +Experiment 014a attempted this previously but was blocked by sandbox test +infrastructure issues and was never benchmarked. The test infrastructure has +since been fixed (experiments 055-056 passed tests). + +## Change Summary +Three `reserve()` additions to `ParallelApplyUtils.cpp`: + +1. **`getReadWriteKeysForStage`**: Reserve `res` unordered_set based on + estimated RW key count (each RW key may have a TTL key, so × 2). Note: + this function runs concurrently with parallel threads, so its impact on + TPS is limited. + +2. **`GlobalParallelApplyLedgerState` constructor**: Reserve `mGlobalEntryMap` + based on total footprint sizes across all stages (RW × 2 + RO × 2 + 1 + per TX for classic source account). + +3. **`collectClusterFootprintEntriesFromGlobal`**: Reserve `mThreadEntryMap` + based on cluster footprint sizes (RW × 2 + RO × 2 per TX in cluster). + +## Results + +### TPS +- Baseline: 18,368 TPS +- Post-change: 18,944 TPS +- Delta: +576 TPS (+3.1%) + +### Tracy Analysis +- `applyLedger` avg: 987ms (baseline: 1,005ms) — **-18ms (-1.8%)** +- `commitChangesFromThread` self-time: 128ms (baseline: 173ms) — **-45ms (-26%)** +- `commitChangesToLedgerTxn` self-time: 120ms (baseline: 164ms) — **-44ms (-27%)** +- `getReadWriteKeysForStage` self-time: 138ms (baseline: 152ms) — **-14ms (-9%)** +- `upsertEntry` cumulative self-time: 425ms (baseline: 446ms) — -21ms (-5%) +- `updateState` self-time: 299ms (baseline: 309ms) — -10ms (noise) +- `addLiveBatch` avg: ~112ms (baseline: ~111ms) — flat + +## Why It Worked +The commit-related functions (`commitChangesFromThread`, `commitChangesToLedgerTxn`) +showed the largest improvements (-26% to -27%) because they merge thread-local +maps into the global map. Without `reserve()`, each merge triggers progressive +rehashing as the destination map grows. With `reserve()`, the destination map +is pre-sized to accommodate all entries, so inserts never trigger rehash. + +The thread-local map reserve in `collectClusterFootprintEntriesFromGlobal` +benefits both the per-TX `upsertEntry` calls (entries insert without rehash) +and the subsequent `commitChangesFromThread` call (the source map is already +properly sized). + +## Files Changed +- `src/transactions/ParallelApplyUtils.cpp` — Added reserve() calls to + getReadWriteKeysForStage, GlobalParallelApplyLedgerState constructor, + and ThreadParallelApplyLedgerState::collectClusterFootprintEntriesFromGlobal + +## Commit +(pending) diff --git a/docs/success/061-move-entries-in-getAllEntries.md b/docs/success/061-move-entries-in-getAllEntries.md new file mode 100644 index 0000000000..0efa93cdfd --- /dev/null +++ b/docs/success/061-move-entries-in-getAllEntries.md @@ -0,0 +1,53 @@ +# Experiment 061: Move entries instead of copying in getAllEntries + +## Date +2026-02-24 + +## Hypothesis +`getAllEntries` deep-copies ~128K+ `LedgerEntry` objects from the `EntryMap` +into three output vectors (init, live, dead) at ~19ms per ledger. Since the +`LedgerTxn` is immediately sealed after `getAllEntries` (the entries are never +accessed again), we can `std::move` the `LedgerEntry` objects instead of +copying them. For XDR-generated types containing `xdr::xvector`, move is O(1) +pointer transfer vs O(N) deep copy. + +The key insight: `LedgerEntryPtr::operator->() const` returns a non-const +`InternalLedgerEntry*`, and `InternalLedgerEntry::ledgerEntry()` has a +non-const overload returning `LedgerEntry&`. So `std::move(entry->ledgerEntry())` +works even through the `EntryMap const&` reference in the existing +`maybeUpdateLastModifiedThenInvokeThenSeal` lambda — no signature changes needed. + +## Change Summary +1. **`LedgerTxn.cpp`**: Changed `getAllEntries` to use + `std::move(entry->ledgerEntry())` in the two `emplace_back` calls for + init and live entries. Added comment explaining the safety rationale + (LedgerTxn is sealed after, entries never accessed again). + +## Results + +### TPS +- Baseline: 18,944 TPS (experiment 060) +- Post-change run 1: 18,688 TPS +- Post-change run 2: 18,368 TPS +- Delta: within noise (exp 059 also showed 18,368/18,944 variance) + +### Tracy Analysis +- `getAllEntries` self-time: 43.7ms → 10.9ms/ledger (baseline 76ms → 19ms/ledger) — **-8.1ms/ledger (-43%)** +- `applyLedger` avg: ~970ms (baseline: ~988ms) — **-18ms/ledger (-1.8%)** +- `addLiveBatch`: 115.3ms/ledger (unchanged — downstream consumers unaffected) +- `updateInMemorySorobanState`: 67.0ms/ledger (baseline: 64ms — within noise) +- `finalize: waitForInMemoryUpdate`: ~0ms (unchanged) +- `finalize: resolveEviction`: 19.8ms/ledger (unchanged) + +## Why TPS Didn't Change +The 8ms saving on the serial path is < 1% of the ~988ms `applyLedger` total. +The binary search resolution at ~18,944 TPS has 128 TPS steps, each adding +~7ms. An 8ms saving is just barely one step, well within the benchmark's +5-10% run-to-run variance. The improvement compounds with other serial path +optimizations. + +## Files Changed +- `src/ledger/LedgerTxn.cpp` — Changed `getAllEntries` to move entries instead + of copying (two `emplace_back` calls changed to use `std::move`) + +## Commit diff --git a/docs/success/063-avoid-building-modifiedkeys-set-eviction.md b/docs/success/063-avoid-building-modifiedkeys-set-eviction.md new file mode 100644 index 0000000000..f8f2e16fa0 --- /dev/null +++ b/docs/success/063-avoid-building-modifiedkeys-set-eviction.md @@ -0,0 +1,62 @@ +# Experiment 063: Avoid Building modifiedKeys Set for Eviction + +## Date +2026-02-24 + +## Hypothesis +`resolveBackgroundEvictionScan` receives an `UnorderedSet` built by +`getAllKeysWithoutSealing()` containing ~128K entries (~20ms to build). However, +the eviction scan only performs ~10-100 lookups into this set (checking whether +eviction candidates have been modified). Building a 128K-entry hash set for +a handful of lookups is wasteful. Direct O(1) lookups into the LedgerTxn's +existing EntryMap would eliminate the set construction entirely. + +## Change Summary +Added `isModifiedKey(LedgerKey const&)` method to `AbstractLedgerTxn` / +`LedgerTxn` that performs an O(1) lookup directly in the LedgerTxn's internal +`mEntry` map. Created two overloads of `resolveBackgroundEvictionScan`: + +1. **Production path** (no set parameter): Uses `ltx.isModifiedKey()` for + direct EntryMap lookups. Called from `LedgerManagerImpl::finalizeLedgerTxnChanges`. +2. **Test path** (with `UnorderedSet` parameter): For test helpers + like `BucketTestUtils` that don't write entries through the LedgerTxn + subsystem and need to provide their own key set. + +The production path completely eliminates the `getAllKeysWithoutSealing()` call +and its ~20ms per-ledger cost. + +## Results + +### TPS +- Baseline: 18,944 TPS +- Run 1: 19,520 TPS +- Run 2: 19,136 TPS +- Average: 19,328 TPS +- Delta: +384 TPS (+2.0%) + +### Tracy Analysis +- `finalize: resolveEviction`: 20ms → 0.116ms/ledger (**99.4% reduction**) +- `getAllKeysWithoutSealing` zone completely eliminated (was ~20ms) +- `resolveBackgroundEvictionScan`: 0.116ms (down from ~20ms) +- Total `applyLedger` improvement dampened because eviction ran partially + concurrently with other work + +## Files Changed +- `src/ledger/LedgerTxn.h` — Added `isModifiedKey` pure virtual to + `AbstractLedgerTxn`, override in `LedgerTxn` +- `src/ledger/LedgerTxnImpl.h` — Added `isModifiedKey` declaration to + `LedgerTxn::Impl` +- `src/ledger/LedgerTxn.cpp` — Added `isModifiedKey` implementation (O(1) + EntryMap lookup via `mEntry.find(InternalLedgerKey(key))`) +- `src/bucket/BucketManager.h` — Added two overloads of + `resolveBackgroundEvictionScan` (production + test) +- `src/bucket/BucketManager.cpp` — Implemented both overloads; production + path uses lambda capturing `ltx.isModifiedKey()` +- `src/ledger/LedgerManagerImpl.cpp` — Removed `getAllKeysWithoutSealing()` + call, uses production overload +- `src/invariant/test/InvariantTests.cpp` — Updated to use production overload +- `src/bucket/test/BucketTestUtils.cpp` — Uses test overload with explicit + key set + +## Commit + diff --git a/scripts/run_apply_load_matrix.py b/scripts/run_apply_load_matrix.py index 2f7bf908d6..dcbfb5345a 100644 --- a/scripts/run_apply_load_matrix.py +++ b/scripts/run_apply_load_matrix.py @@ -18,6 +18,9 @@ DEFAULT_STELLAR_CORE_BIN = SCRIPT_DIR.parent / "src" / "stellar-core" DEFAULT_TEMPLATE_CONFIG = SCRIPT_DIR.parent / "docs" / "apply-load-benchmark-sac.cfg" DEFAULT_OUTPUT_ROOT = Path.home() / "apply-load" +DEFAULT_PERF_BIN = "perf" +DEFAULT_TRACY_CAPTURE_BIN = SCRIPT_DIR.parent / "tracy-capture" +DEFAULT_TRACY_SECONDS = 10 APPLY_LOAD_NUM_LEDGERS = 200 FLOAT_RE = r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)" @@ -69,46 +72,106 @@ def summary(self) -> str: SCENARIOS: tuple[Scenario, ...] = ( + # Scenario( + # model_tx="sac", + # tx_count=6000, + # thread_count=4, + # ), Scenario( model_tx="sac", - tx_count=6400, - thread_count=1, - ), - Scenario( - model_tx="sac", - tx_count=6400, - thread_count=8, - ), - Scenario( - model_tx="custom_token", - tx_count=3000, - thread_count=1, - ), - Scenario( - model_tx="custom_token", - tx_count=3000, + tx_count=6000, thread_count=8, ), + # Scenario( + # model_tx="sac", + # tx_count=3200, + # thread_count=16, + # ), + # Scenario( + # model_tx="sac", + # tx_count=6400, + # thread_count=8, + # ), + # Scenario( + # model_tx="sac", + # tx_count=6400, + # thread_count=16, + # ), + # Scenario( + # model_tx="sac", + # tx_count=6432, + # thread_count=24, + # ), + # Scenario( + # model_tx="custom_token", + # tx_count=3000, + # thread_count=4, + # ), + # Scenario( + # model_tx="custom_token", + # tx_count=3000, + # thread_count=8, + # ), + # Scenario( + # model_tx="soroswap", + # tx_count=2000, + # thread_count=4, + # ), Scenario( model_tx="soroswap", - tx_count=1600, - thread_count=1, - ), - Scenario( - model_tx="soroswap", - tx_count=1600, + tx_count=2000, thread_count=8, ), ) +# SCENARIOS: tuple[Scenario, ...] = ( + # Scenario( + # model_tx="sac", + # tx_count=6400, + # thread_count=8, + # ), + # Scenario( + # model_tx="sac", + # tx_count=6400, + # thread_count=16, + # ), + + + # Scenario( + # model_tx="sac", + # tx_count=3200, + # thread_count=1, + # ), + # Scenario( + # model_tx="sac", + # tx_count=3200, + # thread_count=8, + # ), + # Scenario( + # model_tx="custom_token", + # tx_count=1600, + # thread_count=1, + # ), + # Scenario( + # model_tx="custom_token", + # tx_count=1600, + # thread_count=8, + # ), + # Scenario( + # model_tx="soroswap", + # tx_count=1000, + # thread_count=1, + # ), + # Scenario( + # model_tx="soroswap", + # tx_count=1000, + # thread_count=8, + # ), +# ) def validate_scenarios(scenarios: tuple[Scenario, ...]) -> None: - seen_identifiers: set[str] = set() for scenario in scenarios: identifier = scenario.identifier() - if identifier in seen_identifiers: - raise ValueError(f"Duplicate scenario identifier: {identifier}") - seen_identifiers.add(identifier) if scenario.model_tx != "sac": continue @@ -160,6 +223,36 @@ def parse_args() -> argparse.Namespace: "--build-tag", help="Optional build tag to embed in the run identifier. Defaults to a hash of `stellar-core version` output.", ) + parser.add_argument( + "--profile", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "When enabled, wrap each scenario in `perf record` and write one " + "`.perf.data` file per scenario into the scenario artifact directory." + ), + ) + parser.add_argument( + "--tracy", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "When enabled, run stellar-core in the background and attach " + "`tracy-capture` to collect a Tracy trace file per scenario." + ), + ) + parser.add_argument( + "--tracy-capture-bin", + type=Path, + default=DEFAULT_TRACY_CAPTURE_BIN, + help="Path or name of the tracy-capture binary.", + ) + parser.add_argument( + "--tracy-seconds", + type=int, + default=DEFAULT_TRACY_SECONDS, + help="Number of seconds tracy-capture should record before disconnecting.", + ) return parser.parse_args() @@ -188,6 +281,24 @@ def run_command(command: list[str], *, cwd: Path) -> subprocess.CompletedProcess ) +def resolve_executable_path(executable: Path, *, description: str) -> Path: + expanded = executable.expanduser() + if expanded.exists(): + resolved = expanded.resolve() + else: + resolved_on_path = shutil.which(str(expanded)) + if resolved_on_path is None: + raise FileNotFoundError( + f"{description} not found: {executable} " + f"(also checked {expanded.resolve()})" + ) + resolved = Path(resolved_on_path).resolve() + + if not resolved.is_file(): + raise FileNotFoundError(f"{description} path is not a file: {resolved}") + return resolved + + def get_version_string(stellar_core_bin: Path) -> str: result = run_command([str(stellar_core_bin), "version"], cwd=stellar_core_bin.parent) if result.returncode != 0: @@ -219,6 +330,43 @@ def create_run_id(build_tag: str) -> str: return f"{build_tag}-{timestamp}" +def build_apply_load_command(stellar_core_bin: Path, config_path: Path) -> list[str]: + return [str(stellar_core_bin), "--conf", str(config_path), "apply-load"] + + +def build_perf_record_command( + profiled_command: list[str], perf_data_path: Path +) -> list[str]: + return [ + DEFAULT_PERF_BIN, + "record", + "--freq", + "99", + "--call-graph", + # "dwarf", + "fp", + "--output", + str(perf_data_path), + "--", + *profiled_command, + ] + + +def build_tracy_capture_command( + tracy_capture_bin: Path, tracy_output_path: Path, tracy_seconds: int +) -> list[str]: + return [ + str(tracy_capture_bin), + "-o", + str(tracy_output_path), + "-a", + "127.0.0.1", + "-f", + "-s", + str(tracy_seconds), + ] + + def read_template_config(template_config: Path) -> str: try: return template_config.read_text(encoding="utf-8") @@ -312,18 +460,30 @@ def append_csv_row(results_csv: Path, row: dict[str, str | float]) -> None: writer.writerow(row) -def ensure_inputs(stellar_core_bin: Path, template_config: Path) -> tuple[Path, Path]: - stellar_core_bin = stellar_core_bin.expanduser().resolve() +def ensure_inputs( + stellar_core_bin: Path, + template_config: Path, + *, + profile: bool, + tracy: bool, + tracy_capture_bin: Path, +) -> tuple[Path, Path, Path]: + stellar_core_bin = resolve_executable_path( + stellar_core_bin, description="stellar-core binary" + ) template_config = template_config.expanduser().resolve() + resolved_tracy_capture_bin = tracy_capture_bin.expanduser() - if not stellar_core_bin.exists(): - raise FileNotFoundError(f"stellar-core binary not found: {stellar_core_bin}") - if not stellar_core_bin.is_file(): - raise FileNotFoundError(f"stellar-core path is not a file: {stellar_core_bin}") if not template_config.exists(): raise FileNotFoundError(f"Template config not found: {template_config}") + if profile and shutil.which(DEFAULT_PERF_BIN) is None: + raise FileNotFoundError(f"{DEFAULT_PERF_BIN} not found on PATH") + if tracy: + resolved_tracy_capture_bin = resolve_executable_path( + tracy_capture_bin, description="tracy-capture binary" + ) - return stellar_core_bin, template_config + return stellar_core_bin, template_config, resolved_tracy_capture_bin def run_scenario( @@ -333,36 +493,96 @@ def run_scenario( stellar_core_bin: Path, template_text: str, run_id: str, - logs_dir: Path, + artifacts_dir: Path, + profile: bool, + tracy: bool, + tracy_capture_bin: Path, + tracy_seconds: int, ) -> dict[str, float]: - log_name = f"{run_id}-{scenario_index:02d}-{scenario.slug()}.log" - with tempfile.TemporaryDirectory(prefix=f"apply-load-{scenario.slug()}-") as temp_dir: + slug = scenario.slug() + log_name = f"{run_id}-{scenario_index:02d}-{slug}.log" + perf_name = f"{run_id}-{scenario_index:02d}-{slug}.perf.data" + tracy_name = f"{run_id}-{scenario_index:02d}-{slug}.tracy" + tracy_log_name = f"{run_id}-{scenario_index:02d}-{slug}.tracy-capture.log" + with tempfile.TemporaryDirectory(prefix=f"apply-load-{slug}-") as temp_dir: work_dir = Path(temp_dir) config_text = build_config_text(template_text, scenario, log_name) config_path = work_dir / "apply-load.cfg" config_path.write_text(config_text, encoding="utf-8") + perf_data_path = artifacts_dir / perf_name + tracy_output_path = artifacts_dir / tracy_name + apply_load_command = build_apply_load_command(stellar_core_bin, config_path) + command = apply_load_command + if profile: + command = build_perf_record_command(apply_load_command, perf_data_path) print(f"Running {scenario.summary()}") - result = run_command( - [str(stellar_core_bin), "--conf", str(config_path), "apply-load"], - cwd=work_dir, - ) + if profile: + print(f" Profile data: {perf_data_path}") + if tracy: + print(f" Tracy trace: {tracy_output_path}") + + if tracy: + stdout_path = work_dir / "stdout.txt" + stderr_path = work_dir / "stderr.txt" + with open(stdout_path, "w") as stdout_f, open(stderr_path, "w") as stderr_f: + proc = subprocess.Popen( + command, cwd=work_dir, stdout=stdout_f, stderr=stderr_f, + ) + try: + tracy_command = build_tracy_capture_command( + tracy_capture_bin, tracy_output_path, tracy_seconds, + ) + tracy_result = run_command(tracy_command, cwd=work_dir) + tracy_log_text = "" + if tracy_result.stdout: + tracy_log_text += tracy_result.stdout + if tracy_result.stderr: + tracy_log_text += tracy_result.stderr + if tracy_log_text: + tracy_log_path = artifacts_dir / tracy_log_name + tracy_log_path.write_text(tracy_log_text, encoding="utf-8") + if tracy_result.returncode != 0: + print( + f" Warning: tracy-capture exited with code " + f"{tracy_result.returncode}, see {tracy_log_name}", + file=sys.stderr, + ) + finally: + proc.wait() + stdout_text = stdout_path.read_text(encoding="utf-8", errors="replace") + stderr_text = stderr_path.read_text(encoding="utf-8", errors="replace") + returncode = proc.returncode + else: + result = run_command(command, cwd=work_dir) + stdout_text = result.stdout + stderr_text = result.stderr + returncode = result.returncode scenario_log = work_dir / log_name if scenario_log.exists(): - shutil.copy2(scenario_log, logs_dir / log_name) + shutil.copy2(scenario_log, artifacts_dir / log_name) - if result.returncode != 0: + if returncode != 0: raise RuntimeError( - f"Scenario '{scenario.identifier()}' failed with exit code {result.returncode}.\n" - f"stdout:\n{result.stdout}\n" - f"stderr:\n{result.stderr}" + f"Scenario '{scenario.identifier()}' failed with exit code {returncode}.\n" + f"stdout:\n{stdout_text}\n" + f"stderr:\n{stderr_text}" ) if not scenario_log.exists(): raise RuntimeError( f"Scenario '{scenario.identifier()}' completed but did not produce log file {log_name}" ) + if profile and not perf_data_path.exists(): + raise RuntimeError( + f"Scenario '{scenario.identifier()}' completed but did not produce profile {perf_name}" + ) + if tracy and not tracy_output_path.exists(): + print( + f" Warning: tracy trace file not produced: {tracy_name}", + file=sys.stderr, + ) return parse_benchmark_results(scenario_log) @@ -371,8 +591,12 @@ def main() -> int: args = parse_args() try: - stellar_core_bin, template_config = ensure_inputs( - args.stellar_core_bin, args.template_config + stellar_core_bin, template_config, tracy_capture_bin = ensure_inputs( + args.stellar_core_bin, + args.template_config, + profile=args.profile, + tracy=args.tracy, + tracy_capture_bin=args.tracy_capture_bin, ) scenarios = SCENARIOS validate_scenarios(scenarios) @@ -381,7 +605,7 @@ def main() -> int: run_id = create_run_id(build_tag) output_root = args.output_root.expanduser().resolve() run_dir = output_root / run_id - logs_dir = run_dir / "logs" + artifacts_dir = run_dir / "logs" results_csv = run_dir / "results.csv" stamp_path = run_dir / "stamp" template_text = read_template_config(template_config) @@ -390,7 +614,7 @@ def main() -> int: return 1 try: - logs_dir.mkdir(parents=True, exist_ok=False) + artifacts_dir.mkdir(parents=True, exist_ok=False) except FileExistsError: print(f"Error: run directory already exists: {run_dir}", file=sys.stderr) return 1 @@ -405,12 +629,16 @@ def main() -> int: try: for scenario_index, scenario in enumerate(scenarios, start=1): metrics = run_scenario( - scenario_index, + scenario_index, scenario, stellar_core_bin=stellar_core_bin, template_text=template_text, run_id=run_id, - logs_dir=logs_dir, + artifacts_dir=artifacts_dir, + profile=args.profile, + tracy=args.tracy, + tracy_capture_bin=tracy_capture_bin, + tracy_seconds=args.tracy_seconds, ) append_csv_row( results_csv, diff --git a/src/Makefile.am b/src/Makefile.am index 2eee3584ac..108f7beb31 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,7 +15,7 @@ noinst_HEADERS = $(SRC_H_FILES) # is done by setting the CXXSTDLIB flag, which Rust's C++-building machinery is # sensitive to. Rust passes-on, but does not look inside, CXXFLAGS itself to # realize that it needs this setting. -CXXSTDLIB := $(if $(findstring -stdlib=libc++,$(CXXFLAGS)),c++,$(if $(findstring -stdlib=libstdc++,$(CXXFLAGS)),stdc++,)) +CXXSTDLIB := $(if $(findstring -stdlib=libc++,$(CXXFLAGS)),c++,$(if $(findstring -stdlib=libstdc++,$(CXXFLAGS)),stdc++,stdc++)) if USE_TRACY # NB: this unfortunately long list has to be provided here and kept in sync with @@ -75,7 +75,7 @@ endif # tcmalloc must be linked early to properly override malloc/free stellar_core_LDADD = $(libtcmalloc_LIBS) $(soci_LIBS) $(libmedida_LIBS) \ $(top_builddir)/lib/lib3rdparty.a $(sqlite3_LIBS) \ - $(libpq_LIBS) $(xdrpp_LIBS) $(libsodium_LIBS) + $(libpq_LIBS) $(xdrpp_LIBS) $(libsodium_LIBS) -lcrypto TESTDATA_DIR = testdata TEST_FILES = $(TESTDATA_DIR)/stellar-core_example.cfg $(TESTDATA_DIR)/stellar-core_standalone.cfg \ diff --git a/src/bucket/BucketManager.cpp b/src/bucket/BucketManager.cpp index f190dbabfb..1903f30ee5 100644 --- a/src/bucket/BucketManager.cpp +++ b/src/bucket/BucketManager.cpp @@ -1177,11 +1177,117 @@ BucketManager::startBackgroundEvictionScan(ApplyLedgerStateSnapshot lclSnapshot, "SearchableLiveBucketListSnapshot: eviction scan"); } +EvictedStateVectors +BucketManager::resolveBackgroundEvictionScan( + ApplyLedgerStateSnapshot const& lclSnapshot, AbstractLedgerTxn& ltx) +{ + // Production path: uses direct O(1) lookups in the LedgerTxn's EntryMap + // via isModifiedKey(), avoiding building a full UnorderedSet of all ~128K + // modified keys (~20ms saved per ledger). + auto isModifiedKey = [<x](LedgerKey const& k) { + return ltx.isModifiedKey(k); + }; + + ZoneScoped; + releaseAssert(mEvictionStatistics); + auto timer = mBucketListEvictionMetrics.blockingTime.TimeScope(); + auto ls = LedgerSnapshot(ltx); + auto ledgerSeq = ls.getLedgerHeader().current().ledgerSeq; + auto ledgerVers = ls.getLedgerHeader().current().ledgerVersion; + auto networkConfig = SorobanNetworkConfig::loadFromLedger(ls); + releaseAssert(ledgerSeq == lclSnapshot.getLedgerSeq() + 1); + + if (!mEvictionFuture.valid()) + { + startBackgroundEvictionScan(lclSnapshot, networkConfig); + } + + auto evictionCandidates = mEvictionFuture.get(); + + if (!evictionCandidates->isValid(ledgerSeq, ledgerVers, + networkConfig.stateArchivalSettings())) + { + startBackgroundEvictionScan(lclSnapshot, networkConfig); + evictionCandidates = mEvictionFuture.get(); + } + + auto& eligibleEntries = evictionCandidates->eligibleEntries; + + for (auto iter = eligibleEntries.begin(); iter != eligibleEntries.end();) + { + if (!isModifiedKey(getTTLKey(iter->entry))) + { + if (isModifiedKey(LedgerEntryKey(iter->entry))) + { + auto msg = fmt::format( + "Eviction attempted on modified entry: {}", + xdr::xdr_to_string(LedgerEntryKey(iter->entry))); + CLOG_ERROR(Bucket, "{}", msg); + CLOG_FATAL(Bucket, "{}", REPORT_INTERNAL_BUG); + if (getConfig().INVARIANT_EXTRA_CHECKS) + { + throw std::runtime_error(msg); + } + } + + ++iter; + } + else + { + iter = eligibleEntries.erase(iter); + } + } + + auto remainingEntriesToEvict = + networkConfig.stateArchivalSettings().maxEntriesToArchive; + auto entryToEvictIter = eligibleEntries.begin(); + auto newEvictionIterator = evictionCandidates->endOfRegionIterator; + + std::vector deletedKeys; + std::vector archivedEntries; + + while (remainingEntriesToEvict > 0 && + entryToEvictIter != eligibleEntries.end()) + { + ltx.erase(LedgerEntryKey(entryToEvictIter->entry)); + ltx.erase(getTTLKey(entryToEvictIter->entry)); + --remainingEntriesToEvict; + + if (isTemporaryEntry(entryToEvictIter->entry.data)) + { + deletedKeys.emplace_back(LedgerEntryKey(entryToEvictIter->entry)); + } + else + { + archivedEntries.emplace_back(entryToEvictIter->entry); + } + + deletedKeys.emplace_back(getTTLKey(entryToEvictIter->entry)); + + auto age = ledgerSeq - entryToEvictIter->liveUntilLedger; + mEvictionStatistics->recordEvictedEntry(age); + mBucketListEvictionMetrics.entriesEvicted.inc(); + + newEvictionIterator = entryToEvictIter->iter; + entryToEvictIter = eligibleEntries.erase(entryToEvictIter); + } + + if (remainingEntriesToEvict != 0) + { + newEvictionIterator = evictionCandidates->endOfRegionIterator; + } + + networkConfig.updateEvictionIterator(ltx, newEvictionIterator); + return EvictedStateVectors{deletedKeys, archivedEntries}; +} + EvictedStateVectors BucketManager::resolveBackgroundEvictionScan( ApplyLedgerStateSnapshot const& lclSnapshot, AbstractLedgerTxn& ltx, LedgerKeySet const& modifiedKeys) { + // Test path: uses an explicitly provided key set (for test helpers that + // don't write entries through the LedgerTxn subsystem). ZoneScoped; releaseAssert(mEvictionStatistics); auto timer = mBucketListEvictionMetrics.blockingTime.TimeScope(); @@ -1193,18 +1299,11 @@ BucketManager::resolveBackgroundEvictionScan( if (!mEvictionFuture.valid()) { - // Note: It is safe to begin the eviction scan from an LCL snapshot - // rather than the ledger-state diff (ltx). The scan only proposes - // candidates; this function later validates them by re-checking the - // Soroban config and reloading the latest TTLs. Any entry restored in - // the same ledger will be rejected by eviction validation logic. startBackgroundEvictionScan(lclSnapshot, networkConfig); } auto evictionCandidates = mEvictionFuture.get(); - // If eviction related settings changed during the ledger, we have to - // restart the scan if (!evictionCandidates->isValid(ledgerSeq, ledgerVers, networkConfig.stateArchivalSettings())) { @@ -1216,7 +1315,6 @@ BucketManager::resolveBackgroundEvictionScan( for (auto iter = eligibleEntries.begin(); iter != eligibleEntries.end();) { - // If the TTL has not been modified this ledger, we can evict the entry if (modifiedKeys.find(getTTLKey(iter->entry)) == modifiedKeys.end()) { auto maybeEntryIt = modifiedKeys.find(LedgerEntryKey(iter->entry)); @@ -1246,11 +1344,9 @@ BucketManager::resolveBackgroundEvictionScan( auto entryToEvictIter = eligibleEntries.begin(); auto newEvictionIterator = evictionCandidates->endOfRegionIterator; - // Return vectors include both evicted entry and associated TTL std::vector deletedKeys; std::vector archivedEntries; - // Only actually evict up to maxEntriesToArchive of the eligible entries while (remainingEntriesToEvict > 0 && entryToEvictIter != eligibleEntries.end()) { @@ -1267,7 +1363,6 @@ BucketManager::resolveBackgroundEvictionScan( archivedEntries.emplace_back(entryToEvictIter->entry); } - // Delete TTL for both types deletedKeys.emplace_back(getTTLKey(entryToEvictIter->entry)); auto age = ledgerSeq - entryToEvictIter->liveUntilLedger; @@ -1278,10 +1373,6 @@ BucketManager::resolveBackgroundEvictionScan( entryToEvictIter = eligibleEntries.erase(entryToEvictIter); } - // If remainingEntriesToEvict == 0, that means we could not evict the entire - // scan region, so the new eviction iterator should be after the last entry - // evicted. Otherwise, eviction iterator should be at the end of the scan - // region if (remainingEntriesToEvict != 0) { newEvictionIterator = evictionCandidates->endOfRegionIterator; diff --git a/src/bucket/BucketManager.h b/src/bucket/BucketManager.h index f7a4ce1ffa..6836e76a27 100644 --- a/src/bucket/BucketManager.h +++ b/src/bucket/BucketManager.h @@ -346,6 +346,14 @@ class BucketManager : NonMovableOrCopyable // second vector contains all archived entries (persistent and // ContractCode). Note that when an entry is archived, its TTL key will be // included in the deleted keys vector. + // Production path: checks modified keys via direct O(1) lookups in the + // LedgerTxn's EntryMap, avoiding building a full UnorderedSet. + EvictedStateVectors + resolveBackgroundEvictionScan(ApplyLedgerStateSnapshot const& lclSnapshot, + AbstractLedgerTxn& ltx); + + // Test path: uses an explicitly provided set of modified keys (for test + // helpers that don't write entries through the LedgerTxn subsystem). EvictedStateVectors resolveBackgroundEvictionScan(ApplyLedgerStateSnapshot const& lclSnapshot, AbstractLedgerTxn& ltx, diff --git a/src/bucket/BucketOutputIterator.cpp b/src/bucket/BucketOutputIterator.cpp index 6645f51143..43fd611cd9 100644 --- a/src/bucket/BucketOutputIterator.cpp +++ b/src/bucket/BucketOutputIterator.cpp @@ -168,7 +168,8 @@ template std::shared_ptr BucketOutputIterator::getBucket( BucketManager& bucketManager, MergeKey* mergeKey, - std::unique_ptr> inMemoryState) + std::unique_ptr> inMemoryState, + std::shared_ptr preBuiltIndex) { ZoneScoped; if (mBuf) @@ -219,7 +220,11 @@ BucketOutputIterator::getBucket( if (!index) { - if constexpr (std::is_same_v) + if (preBuiltIndex) + { + index = std::move(preBuiltIndex); + } + else if constexpr (std::is_same_v) { if (inMemoryState) { diff --git a/src/bucket/BucketOutputIterator.h b/src/bucket/BucketOutputIterator.h index a76e1c6bb7..99b42ec2d0 100644 --- a/src/bucket/BucketOutputIterator.h +++ b/src/bucket/BucketOutputIterator.h @@ -55,6 +55,8 @@ template class BucketOutputIterator std::shared_ptr getBucket( BucketManager& bucketManager, MergeKey* mergeKey = nullptr, std::unique_ptr> inMemoryState = + nullptr, + std::shared_ptr preBuiltIndex = nullptr); }; } diff --git a/src/bucket/LiveBucket.cpp b/src/bucket/LiveBucket.cpp index 8101c9d183..5f3f9bd4dc 100644 --- a/src/bucket/LiveBucket.cpp +++ b/src/bucket/LiveBucket.cpp @@ -10,6 +10,7 @@ #include "bucket/BucketOutputIterator.h" #include "bucket/BucketUtils.h" #include "bucket/LedgerCmp.h" +#include #include namespace stellar @@ -383,39 +384,102 @@ LiveBucket::convertToBucketEntry(bool useInit, std::vector const& deadEntries) { ZoneScoped; - std::vector bucket; - bucket.reserve(initEntries.size() + liveEntries.size() + - deadEntries.size()); + // Lightweight reference for indirect sorting: avoids copying and + // swapping full BucketEntry objects (which contain large XDR + // LedgerEntry payloads). Instead we sort small 24-byte ref structs + // and materialise the final BucketEntry vector in one pass. + struct EntryRef + { + BucketEntryType type; + // Exactly one of these is non-null. + LedgerEntry const* livePtr; // for INITENTRY / LIVEENTRY + LedgerKey const* deadPtr; // for DEADENTRY + }; + + size_t totalSize = + initEntries.size() + liveEntries.size() + deadEntries.size(); + + std::vector refs; + refs.reserve(totalSize); + + BucketEntryType initType = useInit ? INITENTRY : LIVEENTRY; for (auto const& e : initEntries) { - BucketEntry ce; - ce.type(useInit ? INITENTRY : LIVEENTRY); - ce.liveEntry() = e; - bucket.push_back(ce); + refs.push_back({initType, &e, nullptr}); } for (auto const& e : liveEntries) { - BucketEntry ce; - ce.type(LIVEENTRY); - ce.liveEntry() = e; - bucket.push_back(ce); + refs.push_back({LIVEENTRY, &e, nullptr}); } for (auto const& e : deadEntries) { - BucketEntry ce; - ce.type(DEADENTRY); - ce.deadEntry() = e; - bucket.push_back(ce); + refs.push_back({DEADENTRY, nullptr, &e}); + } + + // Sort using the same LedgerEntryIdCmp logic but through pointers. + LedgerEntryIdCmp idCmp; + std::sort(refs.begin(), refs.end(), + [&idCmp](EntryRef const& a, EntryRef const& b) { + // METAENTRY sorts below all others; not expected here but + // handled for safety. + if (a.type == METAENTRY || b.type == METAENTRY) + { + return a.type < b.type; + } + + // Compare by ledger-entry identity, same as + // BucketEntryIdCmp::compareLive but using + // pointers into the source vectors. + bool aIsLive = (a.type == LIVEENTRY || a.type == INITENTRY); + bool bIsLive = (b.type == LIVEENTRY || b.type == INITENTRY); + + if (aIsLive && bIsLive) + { + return idCmp(a.livePtr->data, b.livePtr->data); + } + else if (aIsLive && !bIsLive) + { + return idCmp(a.livePtr->data, *b.deadPtr); + } + else if (!aIsLive && bIsLive) + { + return idCmp(*a.deadPtr, b.livePtr->data); + } + else + { + return idCmp(*a.deadPtr, *b.deadPtr); + } + }); + + // Materialise sorted BucketEntry vector in one pass. + std::vector bucket; + bucket.reserve(totalSize); + + for (auto const& r : refs) + { + bucket.emplace_back(); + auto& ce = bucket.back(); + if (r.type == DEADENTRY) + { + ce.type(DEADENTRY); + ce.deadEntry() = *r.deadPtr; + } + else + { + ce.type(r.type); + ce.liveEntry() = *r.livePtr; + } } +#ifndef NDEBUG BucketEntryIdCmp cmp; - std::sort(bucket.begin(), bucket.end(), cmp); releaseAssert(std::adjacent_find( bucket.begin(), bucket.end(), [&cmp](BucketEntry const& lhs, BucketEntry const& rhs) { return !cmp(lhs, rhs); }) == bucket.end()); +#endif return bucket; } @@ -587,29 +651,50 @@ LiveBucket::mergeInMemory(BucketManager& bucketManager, mergedEntries.emplace_back(entry); }; - mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, mc, - shadowIterators, keepShadowedLifecycleEntries); + { + ZoneNamedN(zoneMerge, "mergeInMemory merge", true); + mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, + mc, shadowIterators, keepShadowedLifecycleEntries); + } if (countMergeEvents) { bucketManager.incrMergeCounters(mc); } + // Start index construction on worker thread — reads mergedEntries (const), + // completely independent of the put loop's serialize/hash/write work. + auto indexFuture = std::async(std::launch::async, [&]() { + return std::make_shared(bucketManager, mergedEntries, + meta); + }); + // Write merge output to a bucket and save to disk LiveBucketOutputIterator out(bucketManager.getTmpDir(), /*keepTombstoneEntries=*/true, meta, mc, ctx, doFsync); - for (auto const& e : mergedEntries) { - out.put(e); + ZoneNamedN(zonePut, "mergeInMemory put loop", true); + for (auto const& e : mergedEntries) + { + out.put(e); + } + } + + // Collect the pre-built index + std::shared_ptr preBuiltIndex; + { + ZoneNamedN(zoneWait, "mergeInMemory index future wait", true); + preBuiltIndex = indexFuture.get(); } // Store the merged entries in memory in the new bucket in case this // bucket sees another incoming merge as level 0 curr. return out.getBucket( bucketManager, nullptr, - std::make_unique>(std::move(mergedEntries))); + std::make_unique>(std::move(mergedEntries)), + std::move(preBuiltIndex)); } BucketEntryCounters const& diff --git a/src/crypto/SHA.cpp b/src/crypto/SHA.cpp index 67abe2608b..b22915f306 100644 --- a/src/crypto/SHA.cpp +++ b/src/crypto/SHA.cpp @@ -8,21 +8,33 @@ #include "crypto/Curve25519.h" #include "util/NonCopyable.h" #include -#include +#include + +// Verify that the aligned storage in SHA.h matches the real SHA256_CTX. +static_assert(sizeof(SHA256_CTX) == 112, + "SHA256_CTX size mismatch with aligned storage in SHA.h"); +static_assert(alignof(SHA256_CTX) <= 4, + "SHA256_CTX alignment exceeds aligned storage in SHA.h"); namespace stellar { -// Plain SHA256 +// Helper to access the OpenSSL SHA256_CTX stored in the aligned byte array. +static inline SHA256_CTX* +ctx(std::byte* s) +{ + return reinterpret_cast(s); +} + +// Plain SHA256 — use OpenSSL one-shot (auto-selects SHA-NI on supported CPUs). uint256 sha256(ByteSlice const& bin) { ZoneScoped; uint256 out; - if (crypto_hash_sha256(out.data(), bin.data(), bin.size()) != 0) - { - throw CryptoError("error from crypto_hash_sha256"); - } + // Use the fully-qualified OpenSSL ::SHA256 to avoid name conflict with + // stellar::SHA256 class. + ::SHA256(bin.data(), bin.size(), out.data()); return out; } @@ -43,10 +55,7 @@ SHA256::SHA256() void SHA256::reset() { - if (crypto_hash_sha256_init(&mState) != 0) - { - throw CryptoError("error from crypto_hash_sha256_init"); - } + SHA256_Init(ctx(mState)); mFinished = false; } @@ -58,26 +67,20 @@ SHA256::add(ByteSlice const& bin) { throw std::runtime_error("adding bytes to finished SHA256"); } - if (crypto_hash_sha256_update(&mState, bin.data(), bin.size()) != 0) - { - throw CryptoError("error from crypto_hash_sha256_update"); - } + SHA256_Update(ctx(mState), bin.data(), bin.size()); } uint256 SHA256::finish() { uint256 out; - static_assert(sizeof(out) == crypto_hash_sha256_BYTES, - "unexpected crypto_hash_sha256_BYTES"); + static_assert(sizeof(out) == SHA256_DIGEST_LENGTH, + "unexpected SHA256_DIGEST_LENGTH"); if (mFinished) { throw std::runtime_error("finishing already-finished SHA256"); } - if (crypto_hash_sha256_final(&mState, out.data()) != 0) - { - throw CryptoError("error from crypto_hash_sha256_final"); - } + SHA256_Final(out.data(), ctx(mState)); mFinished = true; return out; } diff --git a/src/crypto/SHA.h b/src/crypto/SHA.h index e00cfd8c66..56ecc92af6 100644 --- a/src/crypto/SHA.h +++ b/src/crypto/SHA.h @@ -6,8 +6,8 @@ #include "crypto/ByteSlice.h" #include "crypto/XDRHasher.h" -#include "sodium/crypto_hash_sha256.h" #include "xdr/Stellar-types.h" +#include #include namespace stellar @@ -21,9 +21,12 @@ uint256 sha256(ByteSlice const& bin); Hash subSha256(ByteSlice const& seed, uint64_t counter); // SHA256 in incremental mode, for large inputs. +// Uses aligned storage for OpenSSL's SHA256_CTX to avoid including +// in this header (which would create a naming conflict +// between OpenSSL's ::SHA256 function and stellar::SHA256 class). class SHA256 { - crypto_hash_sha256_state mState; + alignas(4) std::byte mState[112]; // sizeof(SHA256_CTX) == 112 bool mFinished{false}; public: diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp index 1c92d1c090..a7b4738a15 100644 --- a/src/crypto/SecretKey.cpp +++ b/src/crypto/SecretKey.cpp @@ -18,6 +18,8 @@ #include "util/Math.h" #include "util/RandomEvictionCache.h" #include +#include +#include #include #include #include @@ -41,16 +43,32 @@ namespace stellar // to the state of the process; caching its results centrally // makes all signature-verification in the program faster and // has no effect on correctness. +// +// The cache is sharded across NUM_VERIFY_CACHE_SHARDS shards to +// reduce mutex contention when multiple threads verify signatures +// in parallel. Each shard has its own mutex and cache partition. constexpr size_t VERIFY_SIG_CACHE_SIZE = 250'000; -static std::mutex gVerifySigCacheMutex; -static RandomEvictionCache gVerifySigCache(VERIFY_SIG_CACHE_SIZE); -static uint64_t gVerifyCacheHit = 0; -static uint64_t gVerifyCacheMiss = 0; +constexpr size_t NUM_VERIFY_CACHE_SHARDS = 16; +constexpr size_t VERIFY_SIG_CACHE_SHARD_SIZE = + VERIFY_SIG_CACHE_SIZE / NUM_VERIFY_CACHE_SHARDS; + +struct VerifySigCacheShard +{ + std::mutex mMutex; + RandomEvictionCache mCache; + VerifySigCacheShard() : mCache(VERIFY_SIG_CACHE_SHARD_SIZE) + { + } +}; + +static std::array + gVerifySigCacheShards; +static std::atomic gVerifyCacheHit{0}; +static std::atomic gVerifyCacheMiss{0}; // Global flag to use Rust ed25519-dalek for signature verification -// Protected by gVerifySigCacheMutex -static bool gUseRustDalekVerify = false; +static std::atomic gUseRustDalekVerify{false}; static Hash verifySigCacheKey(PublicKey const& key, Signature const& signature, @@ -322,32 +340,36 @@ SecretKey::fromStrKeySeed(std::string const& strKeySeed) void PubKeyUtils::clearVerifySigCache() { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.clear(); + for (auto& shard : gVerifySigCacheShards) + { + std::lock_guard guard(shard.mMutex); + shard.mCache.clear(); + } } void PubKeyUtils::enableRustDalekVerify() { - std::lock_guard guard(gVerifySigCacheMutex); - gUseRustDalekVerify = true; + gUseRustDalekVerify.store(true, std::memory_order_relaxed); + clearVerifySigCache(); } void PubKeyUtils::seedVerifySigCache(unsigned int seed) { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.seed(seed); + for (size_t i = 0; i < NUM_VERIFY_CACHE_SHARDS; ++i) + { + std::lock_guard guard(gVerifySigCacheShards[i].mMutex); + gVerifySigCacheShards[i].mCache.seed(seed + + static_cast(i)); + } } void PubKeyUtils::flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses) { - std::lock_guard guard(gVerifySigCacheMutex); - hits = gVerifyCacheHit; - misses = gVerifyCacheMiss; - gVerifyCacheHit = 0; - gVerifyCacheMiss = 0; + hits = gVerifyCacheHit.exchange(0, std::memory_order_relaxed); + misses = gVerifyCacheMiss.exchange(0, std::memory_order_relaxed); } std::string @@ -456,24 +478,25 @@ PubKeyUtils::verifySig(PublicKey const& key, Signature const& signature, } auto cacheKey = verifySigCacheKey(key, signature, bin); - bool shouldUseRustDalekVerify; + + // Select shard based on cache key hash to distribute lock contention + auto shardIdx = std::hash{}(cacheKey) % NUM_VERIFY_CACHE_SHARDS; + auto& shard = gVerifySigCacheShards[shardIdx]; { - std::lock_guard guard(gVerifySigCacheMutex); - if (gVerifySigCache.exists(cacheKey)) + std::lock_guard guard(shard.mMutex); + if (auto* cached = shard.mCache.maybeGet(cacheKey)) { - ++gVerifyCacheHit; - std::string hitStr("hit"); - ZoneText(hitStr.c_str(), hitStr.size()); - return {gVerifySigCache.get(cacheKey), - VerifySigCacheLookupResult::HIT}; + gVerifyCacheHit.fetch_add(1, std::memory_order_relaxed); + ZoneText("hit", 3); + return {*cached, VerifySigCacheLookupResult::HIT}; } - - shouldUseRustDalekVerify = gUseRustDalekVerify; } - std::string missStr("miss"); - ZoneText(missStr.c_str(), missStr.size()); + bool shouldUseRustDalekVerify = + gUseRustDalekVerify.load(std::memory_order_relaxed); + + ZoneText("miss", 4); bool ok; if (shouldUseRustDalekVerify) @@ -488,9 +511,11 @@ PubKeyUtils::verifySig(PublicKey const& key, Signature const& signature, key.ed25519().data()) == 0); } - std::lock_guard guard(gVerifySigCacheMutex); - ++gVerifyCacheMiss; - gVerifySigCache.put(cacheKey, ok); + { + std::lock_guard guard(shard.mMutex); + gVerifyCacheMiss.fetch_add(1, std::memory_order_relaxed); + shard.mCache.put(cacheKey, ok); + } return {ok, VerifySigCacheLookupResult::MISS}; } diff --git a/src/herder/TxSetFrame.cpp b/src/herder/TxSetFrame.cpp index 0eb6c4e3c6..55eeb22b48 100644 --- a/src/herder/TxSetFrame.cpp +++ b/src/herder/TxSetFrame.cpp @@ -26,8 +26,12 @@ #include #include +#include +#include +#include #include #include +#include #include namespace stellar @@ -35,6 +39,40 @@ namespace stellar namespace { +#ifdef BUILD_TESTS +double +elapsedMs(std::chrono::steady_clock::time_point const& start) +{ + return std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); +} + +template +auto +measureStage(double* output, Fn&& fn) +{ + auto start = std::chrono::steady_clock::now(); + if constexpr (std::is_void_v>) + { + std::forward(fn)(); + if (output) + { + *output += elapsedMs(start); + } + } + else + { + auto result = std::forward(fn)(); + if (output) + { + *output += elapsedMs(start); + } + return result; + } +} +#endif + std::string getTxSetPhaseName(TxSetPhase phase) { @@ -409,22 +447,161 @@ sortedForApplyParallel(TxStageFrameList const& stages, Hash const& txSetHash) return sortedStages; } +// Create TxFrames from XDR envelopes in parallel. +// Returns nullopt if any transaction has invalid fee. +// Precomputes hashes for all transactions to avoid race conditions in sorting. +std::optional +createTxFramesParallel(Hash const& networkID, + xdr::xvector const& xdrTxs, + size_t maxThreads) +{ + ZoneScoped; + auto const numTxs = xdrTxs.size(); + if (numTxs == 0) + { + return TxFrameList{}; + } + + TxFrameList results(numTxs); + std::atomic validationFailed{false}; + + maxThreads = std::min(numTxs, maxThreads); + if (maxThreads == 0) + { + maxThreads = 1; + } + + auto createTx = [&](size_t index) { + if (validationFailed.load(std::memory_order_relaxed)) + { + return; + } + auto tx = TransactionFrameBase::makeTransactionFromWire(networkID, + xdrTxs[index]); + if (!tx->XDRProvidesValidFee()) + { + validationFailed.store(true, std::memory_order_relaxed); + return; + } + // Precompute hashes to avoid race conditions in sorting checks + (void)tx->getContentsHash(); + (void)tx->getFullHash(); + results[index] = std::move(tx); + }; + + if (maxThreads > 1 && numTxs > 1) + { + // Parallel path: divide work evenly among threads + std::vector> futures; + futures.reserve(maxThreads - 1); + + // Calculate range for each thread + auto processRange = [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) + { + if (validationFailed.load(std::memory_order_relaxed)) + { + return; + } + createTx(i); + } + }; + + size_t itemsPerThread = numTxs / maxThreads; + size_t remainder = numTxs % maxThreads; + + // Spawn maxThreads - 1 workers with their assigned ranges + size_t start = 0; + for (size_t t = 0; t < maxThreads - 1; ++t) + { + size_t count = itemsPerThread + (t < remainder ? 1 : 0); + size_t end = start + count; + futures.emplace_back( + std::async(std::launch::async, processRange, start, end)); + start = end; + } + + // Main thread processes the last range + processRange(start, numTxs); + + for (auto& future : futures) + { + releaseAssert(future.valid()); + try + { + future.get(); + } + catch (std::exception const& e) + { + printErrorAndAbort( + "Exception on parallel TxFrame creation thread: ", + e.what()); + } + catch (...) + { + printErrorAndAbort( + "Unknown exception on parallel TxFrame creation thread"); + } + } + } + else + { + // Sequential path: process all on main thread + for (size_t i = 0; i < numTxs; ++i) + { + createTx(i); + if (validationFailed.load(std::memory_order_relaxed)) + { + break; + } + } + } + + if (validationFailed.load(std::memory_order_relaxed)) + { + return std::nullopt; + } + + return results; +} + bool addWireTxsToList(Hash const& networkID, xdr::xvector const& xdrTxs, - TxFrameList& txList) + TxFrameList& txList, size_t maxThreads) { auto prevSize = txList.size(); txList.reserve(prevSize + xdrTxs.size()); - for (auto const& env : xdrTxs) + + if (xdrTxs.size() >= 2) { - auto tx = TransactionFrameBase::makeTransactionFromWire(networkID, env); - if (!tx->XDRProvidesValidFee()) + // Parallel path for multiple transactions + auto maybeTxs = createTxFramesParallel(networkID, xdrTxs, maxThreads); + if (!maybeTxs) { return false; } - txList.push_back(tx); + txList.insert(txList.end(), std::make_move_iterator(maybeTxs->begin()), + std::make_move_iterator(maybeTxs->end())); } + else + { + // Sequential path for single transaction + for (auto const& env : xdrTxs) + { + auto tx = + TransactionFrameBase::makeTransactionFromWire(networkID, env); + if (!tx->XDRProvidesValidFee()) + { + return false; + } + // Precompute hashes for consistency with parallel path + (void)tx->getContentsHash(); + (void)tx->getFullHash(); + txList.push_back(tx); + } + } + if (!std::is_sorted(txList.begin() + prevSize, txList.end(), &TxSetUtils::hashTxSorter)) { @@ -551,11 +728,22 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app #ifdef BUILD_TESTS , bool enforceTxsApplyOrder, - txtest::ParallelSorobanOrder const& parallelSorobanOrder + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings #endif ) { ZoneScoped; +#ifdef BUILD_TESTS + auto const surgePricingStart = std::chrono::steady_clock::now(); + double* surgePricingField = nullptr; + if (txSetBuildTimings) + { + surgePricingField = phase == TxSetPhase::CLASSIC + ? &txSetBuildTimings->surgePricingClassicMs + : &txSetBuildTimings->surgePricingSorobanMs; + } +#endif auto surgePricingLaneConfig = createSurgePricingLangeConfig(phase, app); std::vector hadTxNotFittingLane; uint32_t ledgerVersion = @@ -603,10 +791,25 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app else { #endif - includedTxs = buildSurgePricedParallelSorobanPhase( - txs, app.getConfig(), - app.getLedgerManager().getLastClosedSorobanNetworkConfig(), - surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion); +#ifdef BUILD_TESTS + includedTxs = measureStage( + txSetBuildTimings + ? &txSetBuildTimings->buildParallelSorobanPhaseMs + : nullptr, + [&]() { + return buildSurgePricedParallelSorobanPhase( + txs, app.getConfig(), + app.getLedgerManager() + .getLastClosedSorobanNetworkConfig(), + surgePricingLaneConfig, hadTxNotFittingLane, + ledgerVersion); + }); +#else + includedTxs = buildSurgePricedParallelSorobanPhase( + txs, app.getConfig(), + app.getLedgerManager().getLastClosedSorobanNetworkConfig(), + surgePricingLaneConfig, hadTxNotFittingLane, ledgerVersion); +#endif #ifdef BUILD_TESTS } #endif @@ -677,6 +880,13 @@ applySurgePricing(TxSetPhase phase, TxFrameList const& txs, Application& app inclusionFeeMap[tx] = laneBaseFee[surgePricingLaneConfig->getLane(*tx)]; }); +#ifdef BUILD_TESTS + if (surgePricingField) + { + *surgePricingField += elapsedMs(surgePricingStart); + } +#endif + return std::make_pair(includedTxs, inclusionFeeMapPtr); } @@ -799,7 +1009,8 @@ makeTxSetFromTransactions( #ifdef BUILD_TESTS , bool skipValidation, - txtest::ParallelSorobanOrder const& parallelSorobanOrder + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings #endif ) { @@ -809,7 +1020,8 @@ makeTxSetFromTransactions( upperBoundCloseTimeOffset, invalidTxs #ifdef BUILD_TESTS , - skipValidation, parallelSorobanOrder + skipValidation, parallelSorobanOrder, + txSetBuildTimings #endif ); } @@ -822,7 +1034,8 @@ makeTxSetFromTransactions( #ifdef BUILD_TESTS , bool skipValidation, - txtest::ParallelSorobanOrder const& parallelSorobanOrder + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings #endif ) { @@ -832,6 +1045,20 @@ makeTxSetFromTransactions( releaseAssert(txPhases.size() <= static_cast(TxSetPhase::PHASE_COUNT)); +#ifdef BUILD_TESTS + auto const totalStart = std::chrono::steady_clock::now(); + if (txSetBuildTimings) + { + *txSetBuildTimings = {}; + } + auto finalizeTimings = [&]() { + if (txSetBuildTimings) + { + txSetBuildTimings->totalMs = elapsedMs(totalStart); + } + }; +#endif + std::vector validatedPhases; UnorderedMap accountFeeMap; for (size_t i = 0; i < txPhases.size(); ++i) @@ -849,63 +1076,84 @@ makeTxSetFromTransactions( auto& invalid = invalidTxs[i]; TxFrameList validatedTxs; #ifdef BUILD_TESTS + double* trimInvalidField = nullptr; + if (txSetBuildTimings) + { + trimInvalidField = expectSoroban + ? &txSetBuildTimings->trimInvalidSorobanMs + : &txSetBuildTimings->trimInvalidClassicMs; + } if (skipValidation) { validatedTxs = phaseTxs; } else { -#endif - validatedTxs = TxSetUtils::trimInvalid( - phaseTxs, app, accountFeeMap, lowerBoundCloseTimeOffset, - upperBoundCloseTimeOffset, invalid); -#ifdef BUILD_TESTS + validatedTxs = measureStage(trimInvalidField, [&]() { + return TxSetUtils::trimInvalid( + phaseTxs, app, accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, invalid); + }); } +#else + validatedTxs = TxSetUtils::trimInvalid( + phaseTxs, app, accountFeeMap, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, invalid); #endif auto phaseType = static_cast(i); - auto [includedTxs, inclusionFeeMapBinding] = - applySurgePricing(phaseType, validatedTxs, app + auto [includedTxs, inclusionFeeMapBinding] = applySurgePricing( + phaseType, validatedTxs, app #ifdef BUILD_TESTS - , - skipValidation, parallelSorobanOrder + , + skipValidation, parallelSorobanOrder, txSetBuildTimings #endif - ); + ); auto inclusionFeeMap = inclusionFeeMapBinding; - std::visit( - [&validatedPhases, phaseType, inclusionFeeMap](auto&& txs) { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - validatedPhases.emplace_back( - TxSetPhaseFrame(phaseType, txs, inclusionFeeMap)); - } - else if constexpr (std::is_same_v) - { - validatedPhases.emplace_back(TxSetPhaseFrame( - phaseType, std::move(txs), inclusionFeeMap)); - } - else - { - // This can't be just `false` as if an assertion is not - // dependent on template argument, it will be - // unconditionally triggered. - static_assert(!std::is_same_v, - "Non-exhaustive visitor"); - } - }, - includedTxs); + if (std::holds_alternative(includedTxs)) + { + validatedPhases.emplace_back( + TxSetPhaseFrame(phaseType, std::get(includedTxs), + inclusionFeeMap)); + } + else if (std::holds_alternative(includedTxs)) + { + validatedPhases.emplace_back(TxSetPhaseFrame( + phaseType, std::get(std::move(includedTxs)), + inclusionFeeMap)); + } + else + { + releaseAssert(false); + } } auto const& lclHeader = app.getLedgerManager().getLastClosedLedgerHeader(); // Preliminary applicable frame - we don't know the contents hash yet, but // we also don't return this. +#ifdef BUILD_TESTS + auto preliminaryApplicableTxSet = measureStage( + txSetBuildTimings ? &txSetBuildTimings->buildApplicableTxSetMs + : nullptr, + [&]() { + return std::unique_ptr( + new ApplicableTxSetFrame(app, lclHeader, validatedPhases, + std::nullopt)); + }); +#else std::unique_ptr preliminaryApplicableTxSet( new ApplicableTxSetFrame(app, lclHeader, validatedPhases, std::nullopt)); +#endif // Do the roundtrip through XDR to ensure we never build an incorrect tx set // for nomination. +#ifdef BUILD_TESTS + auto outputTxSet = measureStage( + txSetBuildTimings ? &txSetBuildTimings->toWireTxSetMs : nullptr, + [&]() { return preliminaryApplicableTxSet->toWireTxSetFrame(); }); +#else auto outputTxSet = preliminaryApplicableTxSet->toWireTxSetFrame(); +#endif #ifdef BUILD_TESTS if (skipValidation) { @@ -913,13 +1161,20 @@ makeTxSetFromTransactions( // and validation flow. preliminaryApplicableTxSet->mContentsHash = outputTxSet->getContentsHash(); + finalizeTimings(); return std::make_pair(outputTxSet, std::move(preliminaryApplicableTxSet)); } #endif - +#ifdef BUILD_TESTS + auto outputApplicableTxSet = measureStage( + txSetBuildTimings ? &txSetBuildTimings->prepareTxSetForApplyMs + : nullptr, + [&]() { return outputTxSet->prepareForApply(app, lclHeader.header); }); +#else ApplicableTxSetFrameConstPtr outputApplicableTxSet = outputTxSet->prepareForApply(app, lclHeader.header); +#endif if (!outputApplicableTxSet) { @@ -929,6 +1184,28 @@ makeTxSetFromTransactions( // Make sure no transactions were lost during the roundtrip and the output // tx set is valid. +#ifdef BUILD_TESTS + bool valid = measureStage( + txSetBuildTimings ? &txSetBuildTimings->validateRoundTripShapeMs + : nullptr, + [&]() { + bool shapeValid = preliminaryApplicableTxSet->numPhases() == + outputApplicableTxSet->numPhases(); + if (shapeValid) + { + for (size_t i = 0; i < preliminaryApplicableTxSet->numPhases(); + ++i) + { + shapeValid = + shapeValid && preliminaryApplicableTxSet->sizeTx( + static_cast(i)) == + outputApplicableTxSet->sizeTx( + static_cast(i)); + } + } + return shapeValid; + }); +#else bool valid = preliminaryApplicableTxSet->numPhases() == outputApplicableTxSet->numPhases(); if (valid) @@ -941,6 +1218,7 @@ makeTxSetFromTransactions( static_cast(i)); } } +#endif if (!valid) { throw std::runtime_error("Created invalid tx set frame - shape is " @@ -948,8 +1226,18 @@ makeTxSetFromTransactions( } // We already trimmed invalid transactions in an earlier call to // `trimInvalid`, so skip transaction validation here +#ifdef BUILD_TESTS + auto validationResult = measureStage( + txSetBuildTimings ? &txSetBuildTimings->validateTxSetMs : nullptr, + [&]() { + return outputApplicableTxSet->checkValidInternalWithResult( + app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, + true); + }); +#else auto validationResult = outputApplicableTxSet->checkValidInternalWithResult( app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, true); +#endif if (validationResult != TxSetValidationResult::VALID) { throw std::runtime_error(fmt::format( @@ -957,6 +1245,9 @@ makeTxSetFromTransactions( toString(validationResult))); } +#ifdef BUILD_TESTS + finalizeTimings(); +#endif return std::make_pair(outputTxSet, std::move(outputApplicableTxSet)); } @@ -998,12 +1289,13 @@ std::pair makeTxSetFromTransactions( TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, bool enforceTxsApplyOrder, - txtest::ParallelSorobanOrder const& parallelSorobanOrder) + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings) { TxFrameList invalid; return makeTxSetFromTransactions( txs, app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, invalid, - enforceTxsApplyOrder, parallelSorobanOrder); + enforceTxsApplyOrder, parallelSorobanOrder, txSetBuildTimings); } std::pair @@ -1011,7 +1303,8 @@ makeTxSetFromTransactions( TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, bool enforceTxsApplyOrder, - txtest::ParallelSorobanOrder const& parallelSorobanOrder) + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings) { releaseAssert(threadIsMain()); releaseAssert(!app.getLedgerManager().isApplying()); @@ -1036,7 +1329,7 @@ makeTxSetFromTransactions( invalid.resize(perPhaseTxs.size()); auto res = makeTxSetFromTransactions( perPhaseTxs, app, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, - invalid, enforceTxsApplyOrder, parallelSorobanOrder); + invalid, enforceTxsApplyOrder, parallelSorobanOrder, txSetBuildTimings); if (enforceTxsApplyOrder) { auto const& resPhases = res.second->getPhases(); @@ -1098,6 +1391,10 @@ TxSetXDRFrame::prepareForApply(Application& app, } #endif ZoneScoped; + + auto const maxThreads = + static_cast(app.getConfig().LEDGER_CLOSE_WORKER_THREADS); + std::vector phaseFrames; if (isGeneralizedTxSet()) { @@ -1114,7 +1411,7 @@ TxSetXDRFrame::prepareForApply(Application& app, { auto maybePhase = TxSetPhaseFrame::makeFromWire( static_cast(phaseId), app.getNetworkID(), - xdrPhases[phaseId]); + xdrPhases[phaseId], maxThreads); if (!maybePhase) { return nullptr; @@ -1126,7 +1423,7 @@ TxSetXDRFrame::prepareForApply(Application& app, { auto const& xdrTxSet = std::get(mXDRTxSet); auto maybePhase = TxSetPhaseFrame::makeFromWireLegacy( - lclHeader, app.getNetworkID(), xdrTxSet.txs); + lclHeader, app.getNetworkID(), xdrTxSet.txs, maxThreads); if (!maybePhase) { return nullptr; @@ -1425,7 +1722,8 @@ TxSetPhaseFrame::Iterator::operator!=(Iterator const& other) const std::optional TxSetPhaseFrame::makeFromWire(TxSetPhase phase, Hash const& networkID, - TransactionPhase const& xdrPhase) + TransactionPhase const& xdrPhase, + size_t maxThreads) { auto inclusionFeeMapPtr = std::make_shared(); auto& inclusionFeeMap = *inclusionFeeMapPtr; @@ -1456,7 +1754,7 @@ TxSetPhaseFrame::makeFromWire(TxSetPhase phase, Hash const& networkID, size_t prevSize = txList.size(); if (!addWireTxsToList(networkID, component.txsMaybeDiscountedFee().txs, - txList)) + txList, maxThreads)) { CLOG_DEBUG(Herder, "Got bad generalized txSet: transactions " @@ -1490,29 +1788,190 @@ TxSetPhaseFrame::makeFromWire(TxSetPhase phase, Hash const& networkID, return std::nullopt; } } - TxStageFrameList stages; - stages.reserve(xdrStages.size()); - for (auto const& xdrStage : xdrStages) + + // Collect all XDR envelopes with their positions for parallel creation + struct TxPosition { - auto& stage = stages.emplace_back(); - stage.reserve(xdrStage.size()); - for (auto const& xdrCluster : xdrStage) + size_t stageIdx; + size_t clusterIdx; + size_t txIdx; + TransactionEnvelope const* env; + }; + std::vector allTxs; + + // Count total transactions and collect positions + size_t totalTxs = 0; + for (size_t s = 0; s < xdrStages.size(); ++s) + { + for (size_t c = 0; c < xdrStages[s].size(); ++c) + { + totalTxs += xdrStages[s][c].size(); + } + } + allTxs.reserve(totalTxs); + + for (size_t s = 0; s < xdrStages.size(); ++s) + { + for (size_t c = 0; c < xdrStages[s].size(); ++c) + { + for (size_t t = 0; t < xdrStages[s][c].size(); ++t) + { + allTxs.push_back({s, c, t, &xdrStages[s][c][t]}); + } + } + } + + // Create TxFrames in parallel + std::vector txFrames(totalTxs); + std::atomic validationFailed{false}; + + if (totalTxs >= 2) + { + size_t effectiveThreads = std::min(totalTxs, maxThreads); + if (effectiveThreads == 0) + { + effectiveThreads = 1; + } + + auto createTx = [&](size_t index) { + if (validationFailed.load(std::memory_order_relaxed)) + { + return; + } + auto tx = TransactionFrameBase::makeTransactionFromWire( + networkID, *allTxs[index].env); + if (!tx->XDRProvidesValidFee()) + { + validationFailed.store(true, std::memory_order_relaxed); + return; + } + // Precompute hashes to avoid race conditions in sorting + (void)tx->getContentsHash(); + (void)tx->getFullHash(); + txFrames[index] = std::move(tx); + }; + + if (effectiveThreads > 1) { - auto& cluster = stage.emplace_back(); - cluster.reserve(xdrCluster.size()); - for (auto const& env : xdrCluster) + // Parallel path: divide work evenly among threads + std::vector> futures; + futures.reserve(effectiveThreads - 1); + + auto processRange = [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) + { + if (validationFailed.load(std::memory_order_relaxed)) + { + return; + } + createTx(i); + } + }; + + size_t itemsPerThread = totalTxs / effectiveThreads; + size_t remainder = totalTxs % effectiveThreads; + + // Spawn effectiveThreads - 1 workers with their assigned ranges + size_t start = 0; + for (size_t t = 0; t < effectiveThreads - 1; ++t) { - auto tx = TransactionFrameBase::makeTransactionFromWire( - networkID, env); - if (!tx->XDRProvidesValidFee()) + size_t count = itemsPerThread + (t < remainder ? 1 : 0); + size_t end = start + count; + futures.emplace_back(std::async(std::launch::async, + processRange, start, end)); + start = end; + } + + // Main thread processes the last range + processRange(start, totalTxs); + + for (auto& future : futures) + { + releaseAssert(future.valid()); + try { - CLOG_DEBUG(Herder, "Got bad generalized txSet: " - "transaction has invalid XDR"); - return std::nullopt; + future.get(); + } + catch (std::exception const& e) + { + printErrorAndAbort( + "Exception on parallel TxFrame creation " + "thread: ", + e.what()); + } + catch (...) + { + printErrorAndAbort( + "Unknown exception on parallel TxFrame creation " + "thread"); + } + } + } + else + { + // Sequential path: process all on main thread + for (size_t i = 0; i < totalTxs; ++i) + { + createTx(i); + if (validationFailed.load(std::memory_order_relaxed)) + { + break; } - cluster.push_back(tx); - inclusionFeeMap[tx] = baseFee; } + } + } + else if (totalTxs == 1) + { + auto tx = TransactionFrameBase::makeTransactionFromWire( + networkID, *allTxs[0].env); + if (!tx->XDRProvidesValidFee()) + { + validationFailed.store(true, std::memory_order_relaxed); + } + else + { + (void)tx->getContentsHash(); + (void)tx->getFullHash(); + txFrames[0] = std::move(tx); + } + } + + if (validationFailed.load(std::memory_order_relaxed)) + { + CLOG_DEBUG( + Herder, + "Got bad generalized txSet: transaction has invalid XDR"); + return std::nullopt; + } + + // Reconstruct the nested structure + TxStageFrameList stages; + stages.reserve(xdrStages.size()); + for (size_t s = 0; s < xdrStages.size(); ++s) + { + stages.emplace_back(); + stages.back().reserve(xdrStages[s].size()); + for (size_t c = 0; c < xdrStages[s].size(); ++c) + { + stages.back().emplace_back(); + stages.back().back().reserve(xdrStages[s][c].size()); + } + } + + // Place TxFrames in their positions and update inclusion fee map + for (size_t i = 0; i < allTxs.size(); ++i) + { + auto const& pos = allTxs[i]; + auto& tx = txFrames[i]; + stages[pos.stageIdx][pos.clusterIdx].push_back(tx); + inclusionFeeMap[tx] = baseFee; + } + + // Verify sorting (fast since hashes are precomputed) + for (auto const& stage : stages) + { + for (auto const& cluster : stage) + { if (!std::is_sorted(cluster.begin(), cluster.end(), &TxSetUtils::hashTxSorter)) { @@ -1558,10 +2017,10 @@ TxSetPhaseFrame::makeFromWire(TxSetPhase phase, Hash const& networkID, std::optional TxSetPhaseFrame::makeFromWireLegacy( LedgerHeader const& lclHeader, Hash const& networkID, - xdr::xvector const& xdrTxs) + xdr::xvector const& xdrTxs, size_t maxThreads) { TxFrameList txList; - if (!addWireTxsToList(networkID, xdrTxs, txList)) + if (!addWireTxsToList(networkID, xdrTxs, txList, maxThreads)) { CLOG_DEBUG( Herder, diff --git a/src/herder/TxSetFrame.h b/src/herder/TxSetFrame.h index de9908645e..82630f6794 100644 --- a/src/herder/TxSetFrame.h +++ b/src/herder/TxSetFrame.h @@ -102,6 +102,23 @@ std::string toString(TxSetValidationResult result); using TxFrameList = std::vector; using PerPhaseTransactionList = std::vector; +#ifdef BUILD_TESTS +struct TxSetBuildPhaseTimings +{ + double totalMs = 0; + double trimInvalidClassicMs = 0; + double surgePricingClassicMs = 0; + double trimInvalidSorobanMs = 0; + double surgePricingSorobanMs = 0; + double buildParallelSorobanPhaseMs = 0; + double buildApplicableTxSetMs = 0; + double toWireTxSetMs = 0; + double prepareTxSetForApplyMs = 0; + double validateRoundTripShapeMs = 0; + double validateTxSetMs = 0; +}; +#endif + // Creates a valid ApplicableTxSetFrame and corresponding TxSetXDRFrame // from the provided transactions. // @@ -124,7 +141,8 @@ makeTxSetFromTransactions( // `enforceTxsApplyOrder` argument in test-only overrides. , bool skipValidation = false, - txtest::ParallelSorobanOrder const& parallelSorobanOrder = {} + txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr #endif ); std::pair @@ -138,7 +156,8 @@ makeTxSetFromTransactions( // `enforceTxsApplyOrder` argument in test-only overrides. , bool skipValidation = false, - txtest::ParallelSorobanOrder const& parallelSorobanOrder = {} + txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr #endif ); @@ -147,13 +166,15 @@ std::pair makeTxSetFromTransactions( TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, bool enforceTxsApplyOrder = false, - txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}); + txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr); std::pair makeTxSetFromTransactions( TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, bool enforceTxsApplyOrder = false, - txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}); + txtest::ParallelSorobanOrder const& parallelSorobanOrder = {}, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr); #endif // `TxSetFrame` is a wrapper around `TransactionSet` or @@ -373,7 +394,8 @@ class TxSetPhaseFrame #ifdef BUILD_TESTS , bool skipValidation, - txtest::ParallelSorobanOrder const& parallelSorobanOrder + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings #endif ); #ifdef BUILD_TESTS @@ -382,7 +404,8 @@ class TxSetPhaseFrame TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, bool enforceTxsApplyOrder, - txtest::ParallelSorobanOrder const& parallelSorobanOrder); + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings); #endif TxSetPhaseFrame(TxSetPhase phase, TxFrameList const& txs, std::shared_ptr inclusionFeeMap); @@ -391,15 +414,21 @@ class TxSetPhaseFrame // Creates a new phase from `TransactionPhase` XDR coming from a // `GeneralizedTransactionSet`. + // maxThreads specifies the maximum number of threads to use for parallel + // TxFrame creation (typically from soroban config + // ledgerMaxDependentTxClusters). static std::optional makeFromWire(TxSetPhase phase, Hash const& networkID, - TransactionPhase const& xdrPhase); + TransactionPhase const& xdrPhase, size_t maxThreads); // Creates a new phase from all the transactions in the legacy // `TransactionSet` XDR. + // maxThreads specifies the maximum number of threads to use for parallel + // TxFrame creation. static std::optional makeFromWireLegacy(LedgerHeader const& lclHeader, Hash const& networkID, - xdr::xvector const& xdrTxs); + xdr::xvector const& xdrTxs, + size_t maxThreads); // Creates a valid empty phase with given `isParallel` flag. static TxSetPhaseFrame makeEmpty(TxSetPhase phase, bool isParallel); @@ -545,7 +574,8 @@ class ApplicableTxSetFrame #ifdef BUILD_TESTS , bool skipValidation, - txtest::ParallelSorobanOrder const& parallelSorobanOrder + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings #endif ); #ifdef BUILD_TESTS @@ -554,7 +584,8 @@ class ApplicableTxSetFrame TxFrameList txs, Application& app, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, TxFrameList& invalidTxs, bool enforceTxsApplyOrder, - txtest::ParallelSorobanOrder const& parallelSorobanOrder); + txtest::ParallelSorobanOrder const& parallelSorobanOrder, + TxSetBuildPhaseTimings* txSetBuildTimings); #endif ApplicableTxSetFrame(Application& app, diff --git a/src/herder/TxSetUtils.cpp b/src/herder/TxSetUtils.cpp index 1b8100f842..bbcd87a846 100644 --- a/src/herder/TxSetUtils.cpp +++ b/src/herder/TxSetUtils.cpp @@ -27,8 +27,10 @@ #include #include +#include #include #include +#include namespace stellar { @@ -58,6 +60,86 @@ removeTxs(TxFrameList const& txs, TxFrameList const& txsToRemove) return newTxs; } + +void +addFeeWithSaturation(UnorderedMap& accountFeeMap, + AccountID const& feeSourceID, int64_t fee) +{ + int64_t& accFee = accountFeeMap[feeSourceID]; + if (INT64_MAX - accFee < fee) + { + accFee = INT64_MAX; + } + else + { + accFee += fee; + } +} + +void +mergeAccountFeeMaps(UnorderedMap& destination, + UnorderedMap const& source) +{ + for (auto const& [feeSourceID, fee] : source) + { + addFeeWithSaturation(destination, feeSourceID, fee); + } +} + +size_t +getValidationThreadCount(size_t txCount, Config const& config) +{ + if (txCount == 0) + { + return 0; + } + + auto const targetThreadCount = + static_cast(config.LEDGER_CLOSE_WORKER_THREADS); + return std::min(txCount, targetThreadCount); +} + +struct ValidationChunkResult +{ + TxFrameList mInvalidTxs; + UnorderedMap mAccountFeeMap; + bool mHadValidationFailure = false; +}; + +void +validateTxChunk(TxFrameList const& txList, size_t chunkBegin, size_t chunkEnd, + AppConnector& appConnector, + LedgerStateSnapshot const& ledgerStateSnapshot, + uint32_t nextLedgerSeq, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + SorobanNetworkConfig const* sorobanConfig, + ValidationChunkResult& chunkResult) +{ + auto diagnostics = DiagnosticEventManager::createDisabled(); + chunkResult.mInvalidTxs.reserve(chunkEnd - chunkBegin); + chunkResult.mAccountFeeMap.reserve(chunkEnd - chunkBegin); + + LedgerSnapshot chunkSnapshot(ledgerStateSnapshot); + chunkSnapshot.getLedgerHeader().currentToModify().ledgerSeq = nextLedgerSeq; + + for (size_t txIndex = chunkBegin; txIndex < chunkEnd; ++txIndex) + { + auto const& tx = txList[txIndex]; + auto txResult = tx->checkValid( + appConnector, chunkSnapshot, 0, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, diagnostics, sorobanConfig); + if (!txResult->isSuccess()) + { + chunkResult.mInvalidTxs.emplace_back(tx); + chunkResult.mHadValidationFailure = true; + } + else + { + addFeeWithSaturation(chunkResult.mAccountFeeMap, + tx->getFeeSourceID(), tx->getFullFee()); + } + } +} } // namespace AccountTransactionQueue::AccountTransactionQueue( @@ -171,10 +253,8 @@ TxSetUtils::getInvalidTxListWithErrors( { ZoneScoped; releaseAssert(threadIsMain()); - LedgerSnapshot ls(app); - // This is done so minSeqLedgerGap is validated against the next - // ledgerSeq, which is what will be used at apply time - ls.getLedgerHeader().currentToModify().ledgerSeq = + auto txList = TxFrameList(txs.begin(), txs.end()); + auto const nextLedgerSeq = app.getLedgerManager().getLastClosedLedgerNum() + 1; TxFrameListWithErrors invalidTxsWithError; @@ -183,67 +263,187 @@ TxSetUtils::getInvalidTxListWithErrors( errorCode = TxSetValidationResult::VALID; std::unordered_set seenInvalidTxs; - auto diagnostics = DiagnosticEventManager::createDisabled(); - for (auto const& tx : txs) + + if (app.getConfig().MODE_USES_IN_MEMORY_LEDGER) { - auto txResult = tx->checkValid(app.getAppConnector(), ls, 0, - lowerBoundCloseTimeOffset, - upperBoundCloseTimeOffset, diagnostics); - if (!txResult->isSuccess()) + LedgerSnapshot ls(app); + ls.getLedgerHeader().currentToModify().ledgerSeq = nextLedgerSeq; + auto const* sorobanConfig = + protocolVersionStartsFrom( + ls.getLedgerHeader().current().ledgerVersion, + SOROBAN_PROTOCOL_VERSION) + ? &app.getLedgerManager().getLastClosedSorobanNetworkConfig() + : nullptr; + auto diagnostics = DiagnosticEventManager::createDisabled(); + for (auto const& tx : txList) { - invalidTxs.emplace_back(tx); - seenInvalidTxs.emplace(tx->getFullHash()); - errorCode = TxSetValidationResult::TX_VALIDATION_FAILED; - } - else - { - int64_t& accFee = accountFeeMap[tx->getFeeSourceID()]; - if (INT64_MAX - accFee < tx->getFullFee()) + auto txResult = tx->checkValid( + app.getAppConnector(), ls, 0, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, diagnostics, sorobanConfig); + if (!txResult->isSuccess()) { - accFee = INT64_MAX; + invalidTxs.emplace_back(tx); + seenInvalidTxs.emplace(tx->getFullHash()); + errorCode = TxSetValidationResult::TX_VALIDATION_FAILED; } else { - accFee += tx->getFullFee(); + addFeeWithSaturation(accountFeeMap, tx->getFeeSourceID(), + tx->getFullFee()); } } } - - auto header = ls.getLedgerHeader().current(); - for (auto const& tx : txs) + else { - // Already added invalid tx - if (seenInvalidTxs.find(tx->getFullHash()) != seenInvalidTxs.end()) - { - continue; - } + auto const ledgerStateSnapshot = + app.getLedgerManager().copyLedgerStateSnapshot(); + LedgerSnapshot ls(ledgerStateSnapshot); + // This is done so minSeqLedgerGap is validated against the next + // ledgerSeq, which is what will be used at apply time + ls.getLedgerHeader().currentToModify().ledgerSeq = nextLedgerSeq; + auto const* sorobanConfig = + protocolVersionStartsFrom( + ls.getLedgerHeader().current().ledgerVersion, + SOROBAN_PROTOCOL_VERSION) + ? &app.getLedgerManager().getLastClosedSorobanNetworkConfig() + : nullptr; - auto feeSourceID = tx->getFeeSourceID(); - auto feeSource = ls.getAccount(feeSourceID); - // feeSource should exist since we've already run checkValid, log - // internal bug - if (!feeSource) + auto const numThreads = + getValidationThreadCount(txList.size(), app.getConfig()); + if (numThreads != 0) { - CLOG_ERROR(Herder, - "Account not found when checking TxSet validity"); - CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); - continue; + std::vector validationResults(numThreads); + auto const baseChunkSize = txList.size() / numThreads; + auto const extraTxs = txList.size() % numThreads; + if (numThreads == 1) + { + validateTxChunk(txList, 0, txList.size(), app.getAppConnector(), + ledgerStateSnapshot, nextLedgerSeq, + lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, sorobanConfig, + validationResults[0]); + } + else + { + std::vector validationExceptions( + numThreads); + std::vector threads; + threads.reserve(numThreads); + + size_t chunkBegin = 0; + for (size_t threadIndex = 0; threadIndex < numThreads; + ++threadIndex) + { + auto const chunkSize = + baseChunkSize + (threadIndex < extraTxs ? 1u : 0u); + auto const chunkEnd = chunkBegin + chunkSize; + threads.emplace_back( + [&, threadIndex, chunkBegin, chunkEnd]() { + try + { + validateTxChunk( + txList, chunkBegin, chunkEnd, + app.getAppConnector(), ledgerStateSnapshot, + nextLedgerSeq, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, sorobanConfig, + validationResults[threadIndex]); + } + catch (...) + { + validationExceptions[threadIndex] = + std::current_exception(); + } + }); + + chunkBegin = chunkEnd; + } + + for (auto& thread : threads) + { + thread.join(); + } + + for (auto const& validationException : validationExceptions) + { + if (validationException) + { + std::rethrow_exception(validationException); + } + } + } + + for (auto& validationResult : validationResults) + { + if (validationResult.mHadValidationFailure) + { + errorCode = TxSetValidationResult::TX_VALIDATION_FAILED; + } + + for (auto const& invalidTx : validationResult.mInvalidTxs) + { + invalidTxs.emplace_back(invalidTx); + seenInvalidTxs.emplace(invalidTx->getFullHash()); + } + + mergeAccountFeeMaps(accountFeeMap, + validationResult.mAccountFeeMap); + } } - auto it = accountFeeMap.find(feeSourceID); - auto totFee = it->second; - if (getAvailableBalance(header, feeSource.current()) < totFee) + } + + auto validateFeeBalances = [&](LedgerSnapshot& ls) { + auto header = ls.getLedgerHeader().current(); + for (auto const& tx : txList) { - invalidTxs.push_back(tx); - // Only override the error code if it wasn't already set - if (errorCode == TxSetValidationResult::VALID) + // Already added invalid tx + if (seenInvalidTxs.find(tx->getFullHash()) != seenInvalidTxs.end()) { - errorCode = TxSetValidationResult::ACCOUNT_CANT_PAY_FEE; + continue; + } + + auto feeSourceID = tx->getFeeSourceID(); + auto feeSource = ls.getAccount(feeSourceID); + // feeSource should exist since we've already run checkValid, log + // internal bug + if (!feeSource) + { + CLOG_ERROR(Herder, + "Account not found when checking TxSet validity"); + CLOG_ERROR(Herder, "{}", REPORT_INTERNAL_BUG); + continue; + } + auto it = accountFeeMap.find(feeSourceID); + auto totFee = it->second; + if (getAvailableBalance(header, feeSource.current()) < totFee) + { + invalidTxs.push_back(tx); + // Only override the error code if it wasn't already set + if (errorCode == TxSetValidationResult::VALID) + { + errorCode = TxSetValidationResult::ACCOUNT_CANT_PAY_FEE; + } + releaseAssert(seenInvalidTxs.insert(tx->getFullHash()).second); + CLOG_DEBUG(Herder, + "Got bad txSet: account can't pay fee tx: {}", + xdrToCerealString(tx->getEnvelope(), + "TransactionEnvelope")); } - releaseAssert(seenInvalidTxs.insert(tx->getFullHash()).second); - CLOG_DEBUG( - Herder, "Got bad txSet: account can't pay fee tx: {}", - xdrToCerealString(tx->getEnvelope(), "TransactionEnvelope")); } + }; + + if (app.getConfig().MODE_USES_IN_MEMORY_LEDGER) + { + LedgerSnapshot ls(app); + ls.getLedgerHeader().currentToModify().ledgerSeq = nextLedgerSeq; + validateFeeBalances(ls); + } + else + { + auto const ledgerStateSnapshot = + app.getLedgerManager().copyLedgerStateSnapshot(); + LedgerSnapshot ls(ledgerStateSnapshot); + ls.getLedgerHeader().currentToModify().ledgerSeq = nextLedgerSeq; + validateFeeBalances(ls); } return invalidTxsWithError; diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index 7ea0fe76c3..5890561ebe 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -976,6 +976,45 @@ TEST_CASE("getInvalidTxListWithErrors returns no duplicates") REQUIRE(invalidTxs.size() == 3); } +TEST_CASE("getInvalidTxListWithErrors reduces fee maps") +{ + Config cfg(getTestConfig()); + VirtualClock clock; + Application::pointer app = createTestApplication(clock, cfg); + + auto const minBalance2 = app->getLedgerManager().getLastMinBalance(2); + auto root = app->getRoot(); + + TxFrameList txs; + txs.reserve(33); + + int64_t expectedAddedFee = 0; + auto feeSource = root->create("fee-src", minBalance2 + 100'000); + auto unrelatedAccount = root->create("other", minBalance2); + for (size_t i = 0; i < 33; ++i) + { + auto source = root->create(fmt::format("src-{}", i), minBalance2); + auto innerTx = transactionFromOperations( + *app, source, source.getLastSequenceNumber() + 1, + {payment(source.getPublicKey(), 1)}, 100); + auto feeBumpTx = feeBump(*app, feeSource, innerTx, 200); + expectedAddedFee += feeBumpTx->getFullFee(); + txs.emplace_back(feeBumpTx); + } + + UnorderedMap accountFeeMap; + accountFeeMap[feeSource.getPublicKey()] = 123; + accountFeeMap[unrelatedAccount.getPublicKey()] = 456; + + auto [invalidTxs, result] = + TxSetUtils::getInvalidTxListWithErrors(txs, *app, accountFeeMap, 0, 0); + + REQUIRE(result == TxSetValidationResult::VALID); + REQUIRE(invalidTxs.empty()); + REQUIRE(accountFeeMap[feeSource.getPublicKey()] == 123 + expectedAddedFee); + REQUIRE(accountFeeMap[unrelatedAccount.getPublicKey()] == 456); +} + TEST_CASE("txset", "[herder][txset]") { SECTION("generalized tx set protocol") diff --git a/src/invariant/test/InvariantTests.cpp b/src/invariant/test/InvariantTests.cpp index a8de5500ec..0f5b050b22 100644 --- a/src/invariant/test/InvariantTests.cpp +++ b/src/invariant/test/InvariantTests.cpp @@ -405,7 +405,7 @@ TEST_CASE_VERSIONS("State archival eviction invariant", "[invariant][archival]") ltx.loadHeader().current().ledgerSeq++; auto evictedState = app->getBucketManager().resolveBackgroundEvictionScan(applySnap, - ltx, {}); + ltx); applySnap = app->getLedgerManager().copyApplyLedgerStateSnapshot(); @@ -668,9 +668,11 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") LedgerEntry modifiedEntry = *entryData.ledgerEntry; modifiedEntry.lastModifiedLedgerSeq += 100; auto ttlData = entryData.ttlData; + auto sizeBytes = entryData.sizeBytes; modifiedState.mContractDataEntries.erase(it); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(modifiedEntry, ttlData)); + InternalContractDataMapEntry(modifiedEntry, ttlData, + sizeBytes)); } auto result = @@ -711,7 +713,8 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") createContractDataWithTTL(PERSISTENT, 1000); TTLData ttlData(extraTTL.data.ttl().liveUntilLedgerSeq, 1); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(extraEntry, ttlData)); + InternalContractDataMapEntry(extraEntry, ttlData, + xdr::xdr_size(extraEntry))); } auto result = @@ -741,8 +744,8 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") TTLData wrongTTL(42, 1); modifiedState.mContractDataEntries.erase(it); - modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(entryCopy, wrongTTL)); + modifiedState.mContractDataEntries.emplace(InternalContractDataMapEntry( + entryCopy, wrongTTL, entryData.sizeBytes)); auto result = invariant.checkSnapshot(makeSnap(), modifiedState, noopIsStopping); diff --git a/src/ledger/InMemorySorobanState.cpp b/src/ledger/InMemorySorobanState.cpp index ded1ec1bcf..6be77a8e41 100644 --- a/src/ledger/InMemorySorobanState.cpp +++ b/src/ledger/InMemorySorobanState.cpp @@ -8,6 +8,7 @@ #include "ledger/LedgerTypeUtils.h" #include "ledger/SorobanMetrics.h" #include "util/GlobalChecks.h" +#include #include #include @@ -57,9 +58,10 @@ InMemorySorobanState::updateContractDataTTL( { // Since entries are immutable, we must erase and re-insert auto ledgerEntryPtr = dataIt->get().ledgerEntry; + auto sizeBytes = dataIt->get().sizeBytes; mContractDataEntries.erase(dataIt); - mContractDataEntries.emplace( - InternalContractDataMapEntry(std::move(ledgerEntryPtr), newTtlData)); + mContractDataEntries.emplace(InternalContractDataMapEntry( + std::move(ledgerEntryPtr), newTtlData, sizeBytes)); } void @@ -99,7 +101,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) releaseAssertOrThrow(dataIt != mContractDataEntries.end()); releaseAssertOrThrow(dataIt->get().ledgerEntry != nullptr); - uint32_t oldSize = xdr::xdr_size(*dataIt->get().ledgerEntry); + uint32_t oldSize = dataIt->get().sizeBytes; uint32_t newSize = xdr::xdr_size(ledgerEntry); updateStateSizeOnEntryUpdate(oldSize, newSize, /*isContractCode=*/false); @@ -107,7 +109,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) auto preservedTTL = dataIt->get().ttlData; mContractDataEntries.erase(dataIt); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, preservedTTL)); + InternalContractDataMapEntry(ledgerEntry, preservedTTL, newSize)); } void @@ -135,10 +137,10 @@ InMemorySorobanState::createContractDataEntry(LedgerEntry const& ledgerEntry) } // else: TTL hasn't arrived yet, initialize to 0 (will be updated later) - updateStateSizeOnEntryUpdate(0, xdr::xdr_size(ledgerEntry), - /*isContractCode=*/false); + uint32_t sizeBytes = xdr::xdr_size(ledgerEntry); + updateStateSizeOnEntryUpdate(0, sizeBytes, /*isContractCode=*/false); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, ttlData)); + InternalContractDataMapEntry(ledgerEntry, ttlData, sizeBytes)); } bool @@ -196,7 +198,7 @@ InMemorySorobanState::deleteContractData(LedgerKey const& ledgerKey) mContractDataEntries.find(InternalContractDataMapEntry(ledgerKey)); releaseAssertOrThrow(it != mContractDataEntries.end()); releaseAssertOrThrow(it->get().ledgerEntry != nullptr); - updateStateSizeOnEntryUpdate(xdr::xdr_size(*it->get().ledgerEntry), 0, + updateStateSizeOnEntryUpdate(it->get().sizeBytes, 0, /*isContractCode=*/false); mContractDataEntries.erase(it); } @@ -540,6 +542,7 @@ InMemorySorobanState::updateState( std::optional const& sorobanConfig, SorobanMetrics& metrics) { + ZoneScoped; // After initialization, we must apply every ledger in order to the // in-memory state with no gaps. releaseAssertOrThrow(mLastClosedLedgerSeq + 1 == lh.ledgerSeq); diff --git a/src/ledger/InMemorySorobanState.h b/src/ledger/InMemorySorobanState.h index 0a85aa4840..c42839021b 100644 --- a/src/ledger/InMemorySorobanState.h +++ b/src/ledger/InMemorySorobanState.h @@ -45,14 +45,20 @@ struct TTLData // ContractDataMapEntryT stores a ContractData LedgerEntry and its TTL. TTL is // stored directly with the data to avoid an additional lookup and save memory. +// We also cache the XDR size to avoid repeated xdr_size() calls during updates. struct ContractDataMapEntryT { std::shared_ptr const ledgerEntry; TTLData const ttlData; + // Cached XDR serialized size to avoid repeated xdr_size() calls + uint32_t const sizeBytes; explicit ContractDataMapEntryT( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : ledgerEntry(std::move(ledgerEntry)), ttlData(ttlData) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : ledgerEntry(std::move(ledgerEntry)) + , ttlData(ttlData) + , sizeBytes(sizeBytes) { } }; @@ -131,8 +137,6 @@ class InternalContractDataMapEntry } }; - // ValueEntry stores actual ContractData entries in the map. - // Contains both the LedgerEntry and its TTL information. struct ValueEntry : public AbstractEntry { private: @@ -140,8 +144,8 @@ class InternalContractDataMapEntry public: ValueEntry(std::shared_ptr&& ledgerEntry, - TTLData ttlData) - : entry(std::move(ledgerEntry), ttlData) + TTLData ttlData, uint32_t sizeBytes) + : entry(std::move(ledgerEntry), ttlData, sizeBytes) { } @@ -169,7 +173,7 @@ class InternalContractDataMapEntry { return std::make_unique( std::make_shared(*entry.ledgerEntry), - entry.ttlData); + entry.ttlData, entry.sizeBytes); } }; @@ -223,16 +227,19 @@ class InternalContractDataMapEntry // Creates a ValueEntry from a LedgerEntry (copies the entry) InternalContractDataMapEntry(LedgerEntry const& ledgerEntry, - TTLData ttlData) + TTLData ttlData, uint32_t sizeBytes) : impl(std::make_unique( - std::make_shared(ledgerEntry), ttlData)) + std::make_shared(ledgerEntry), ttlData, + sizeBytes)) { } // Creates a ValueEntry from a shared_ptr (avoids copying) InternalContractDataMapEntry( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : impl(std::make_unique(std::move(ledgerEntry), ttlData)) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : impl(std::make_unique(std::move(ledgerEntry), ttlData, + sizeBytes)) { } diff --git a/src/ledger/InternalLedgerEntry.cpp b/src/ledger/InternalLedgerEntry.cpp index c513645f14..132991ec0d 100644 --- a/src/ledger/InternalLedgerEntry.cpp +++ b/src/ledger/InternalLedgerEntry.cpp @@ -474,6 +474,12 @@ InternalLedgerEntry::InternalLedgerEntry(LedgerEntry const& le) ledgerEntry() = le; } +InternalLedgerEntry::InternalLedgerEntry(LedgerEntry&& le) + : InternalLedgerEntry(InternalLedgerEntryType::LEDGER_ENTRY) +{ + ledgerEntry() = std::move(le); +} + InternalLedgerEntry::InternalLedgerEntry(SponsorshipEntry const& se) : InternalLedgerEntry(InternalLedgerEntryType::SPONSORSHIP) { diff --git a/src/ledger/InternalLedgerEntry.h b/src/ledger/InternalLedgerEntry.h index b12bfaaa68..6146d1caf4 100644 --- a/src/ledger/InternalLedgerEntry.h +++ b/src/ledger/InternalLedgerEntry.h @@ -140,6 +140,7 @@ class InternalLedgerEntry explicit InternalLedgerEntry(InternalLedgerEntryType t); InternalLedgerEntry(LedgerEntry const& le); + InternalLedgerEntry(LedgerEntry&& le); explicit InternalLedgerEntry(SponsorshipEntry const& se); explicit InternalLedgerEntry(SponsorshipCounterEntry const& sce); explicit InternalLedgerEntry(MaxSeqNumToApplyEntry const& msne); diff --git a/src/ledger/LedgerEntryScope.cpp b/src/ledger/LedgerEntryScope.cpp index 9d9fde38e0..3fe4e13baa 100644 --- a/src/ledger/LedgerEntryScope.cpp +++ b/src/ledger/LedgerEntryScope.cpp @@ -277,6 +277,13 @@ ScopedLedgerEntryOpt::modifyInScope( scope.scopeModifyOptionalEntry(*this, func); } +template +std::optional +ScopedLedgerEntryOpt::moveFromScope(LedgerEntryScope const& scope) +{ + return scope.scopeMoveOptionalEntry(*this); +} + template bool ScopedLedgerEntryOpt::operator==(ScopedLedgerEntryOpt const& other) const @@ -395,6 +402,19 @@ LedgerEntryScope::scopeModifyOptionalEntry( func(w.mEntry); } +template +std::optional +LedgerEntryScope::scopeMoveOptionalEntry(ScopedLedgerEntryOpt& w) const +{ + if (w.mScopeID != mScopeID) + { + throw std::runtime_error(fmt::format( + "scopeMoveOptionalEntry: scope ID '{}' != entry scope ID '{}'", + mScopeID, w.mScopeID)); + } + return std::move(w.mEntry); +} + template ScopedLedgerEntry LedgerEntryScope::scopeAdoptEntry(LedgerEntry&& entry) const @@ -417,6 +437,14 @@ LedgerEntryScope::scopeAdoptEntryOpt( return ScopedLedgerEntryOpt(mScopeID, entry); } +template +ScopedLedgerEntryOpt +LedgerEntryScope::scopeAdoptEntryOpt( + std::optional&& entry) const +{ + return ScopedLedgerEntryOpt(mScopeID, std::move(entry)); +} + template template ScopedLedgerEntry @@ -438,6 +466,23 @@ LedgerEntryScope::scopeAdoptEntryFromImpl( return EntryT{mScopeID, entry.mEntry}; } +template +template +ScopedLedgerEntry +LedgerEntryScope::scopeAdoptEntryFromImpl( + ScopedLedgerEntry&& entry, + LedgerEntryScope const& scope) const +{ + if (scope.mActive) + { + throw std::runtime_error(fmt::format( + "scopeAdoptEntryFrom: adopting entry with scope ID {} from " + "still-active scope ID '{}'", + entry.mScopeID, scope.mScopeID)); + } + return EntryT{mScopeID, std::move(entry.mEntry)}; +} + template template ScopedLedgerEntryOpt @@ -456,6 +501,24 @@ LedgerEntryScope::scopeAdoptEntryOptFromImpl( return ScopedLedgerEntryOpt{mScopeID, entry.mEntry}; } +template +template +ScopedLedgerEntryOpt +LedgerEntryScope::scopeAdoptEntryOptFromImpl( + ScopedLedgerEntryOpt&& entry, + LedgerEntryScope const& scope) const +{ + if (scope.mActive) + { + throw std::runtime_error( + fmt::format("scopeAdoptEntryOptFrom: adopting entry with " + "scope ID {} from " + "still-active scope ID '{}'", + entry.mScopeID, scope.mScopeID)); + } + return ScopedLedgerEntryOpt{mScopeID, std::move(entry.mEntry)}; +} + ///////////////////////////////// // DeactivateScopeGuard ///////////////////////////////// @@ -495,6 +558,20 @@ FOREACH_STATIC_LEDGER_ENTRY_SCOPE(INSTANTIATE_SCOPE_CLASSES) scopeAdoptEntryOptFromImpl( \ ScopedLedgerEntryOpt const&, \ LedgerEntryScope const&) \ + const; \ +\ + template ScopedLedgerEntry \ + LedgerEntryScope:: \ + scopeAdoptEntryFromImpl( \ + ScopedLedgerEntry&&, \ + LedgerEntryScope const&) \ + const; \ +\ + template ScopedLedgerEntryOpt \ + LedgerEntryScope:: \ + scopeAdoptEntryOptFromImpl( \ + ScopedLedgerEntryOpt&&, \ + LedgerEntryScope const&) \ const; FOR_EACH_VALID_SCOPE_ADOPTION(INSTANTIATE_ADOPT_METHODS) diff --git a/src/ledger/LedgerEntryScope.h b/src/ledger/LedgerEntryScope.h index 7b5b59b1ac..9503fcfb26 100644 --- a/src/ledger/LedgerEntryScope.h +++ b/src/ledger/LedgerEntryScope.h @@ -310,6 +310,10 @@ template class ScopedLedgerEntryOpt readInScope(LedgerEntryScope const& scope) const; void modifyInScope(LedgerEntryScope const& scope, std::function&)> func); + // Move the entry out of the scoped wrapper, leaving it in a moved-from + // state. This is only safe when the scoped state will not be accessed + // again (e.g., during final consumption of a GlobalParallelApplyState). + std::optional moveFromScope(LedgerEntryScope const& scope); bool operator==(ScopedLedgerEntryOpt const& other) const; bool operator<(ScopedLedgerEntryOpt const& other) const; @@ -382,11 +386,13 @@ template class LedgerEntryScope void scopeModifyOptionalEntry( OptionalEntryT& w, std::function&)> func) const; + std::optional scopeMoveOptionalEntry(OptionalEntryT& w) const; EntryT scopeAdoptEntry(LedgerEntry&& entry) const; EntryT scopeAdoptEntry(LedgerEntry const& entry) const; OptionalEntryT scopeAdoptEntryOpt(std::optional const& entry) const; + OptionalEntryT scopeAdoptEntryOpt(std::optional&& entry) const; template EntryT @@ -414,6 +420,32 @@ template class LedgerEntryScope return scopeAdoptEntryOptFromImpl(entry, scope); } + template + EntryT + scopeAdoptEntryFrom(ScopedLedgerEntry&& entry, + LedgerEntryScope const& scope) const + { + static_assert( + IsValidScopeAdoption::value, + "Invalid scope adoption: this transition is not allowed. " + "Check FOR_EACH_VALID_SCOPE_ADOPTION in LedgerEntryScope.h " + "for the list of valid transitions."); + return scopeAdoptEntryFromImpl(std::move(entry), scope); + } + + template + OptionalEntryT + scopeAdoptEntryOptFrom(ScopedLedgerEntryOpt&& entry, + LedgerEntryScope const& scope) const + { + static_assert( + IsValidScopeAdoption::value, + "Invalid scope adoption: this transition is not allowed. " + "Check FOR_EACH_VALID_SCOPE_ADOPTION in LedgerEntryScope.h " + "for the list of valid transitions."); + return scopeAdoptEntryOptFromImpl(std::move(entry), scope); + } + private: template EntryT @@ -424,6 +456,16 @@ template class LedgerEntryScope OptionalEntryT scopeAdoptEntryOptFromImpl(ScopedLedgerEntryOpt const& entry, LedgerEntryScope const& scope) const; + + template + EntryT + scopeAdoptEntryFromImpl(ScopedLedgerEntry&& entry, + LedgerEntryScope const& scope) const; + + template + OptionalEntryT + scopeAdoptEntryOptFromImpl(ScopedLedgerEntryOpt&& entry, + LedgerEntryScope const& scope) const; }; template class DeactivateScopeGuard diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 4c14006952..ff4199057b 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -74,6 +74,7 @@ #include "LedgerManagerImpl.h" #include +#include #include #include #include @@ -244,6 +245,14 @@ LedgerManagerImpl::ApplyState::getInMemorySorobanState() const return mInMemorySorobanState; } +InMemorySorobanState& +LedgerManagerImpl::ApplyState::getInMemorySorobanStateForUpdate() +{ + releaseAssert(mPhase == Phase::SETTING_UP_STATE || + mPhase == Phase::COMMITTING); + return mInMemorySorobanState; +} + #ifdef BUILD_TESTS InMemorySorobanState& LedgerManagerImpl::ApplyState::getInMemorySorobanStateForTesting() @@ -864,6 +873,12 @@ LedgerManagerImpl::getExpectedLedgerCloseTime() const } #ifdef BUILD_TESTS +LedgerManagerImpl::LedgerClosePhaseTimings const& +LedgerManagerImpl::getLastPhaseTimings() const +{ + return mLastPhaseTimings; +} + std::vector const& LedgerManagerImpl::getLastClosedLedgerTxMeta() { @@ -1560,7 +1575,16 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, header.current().scpValue = sv; maybeResetLedgerCloseMetaDebugStream(header.current().ledgerSeq); +#ifdef BUILD_TESTS + auto phaseStart = std::chrono::steady_clock::now(); +#endif auto applicableTxSet = txSet->prepareForApply(mApp, prevHeader); +#ifdef BUILD_TESTS + auto phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.prepareTxSetMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif if (applicableTxSet == nullptr) { @@ -1596,8 +1620,9 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, } #ifdef BUILD_TESTS - // We always store the ledgerCloseMeta in tests so we can inspect it. - if (!ledgerCloseMeta) + // We always store the ledgerCloseMeta in tests so we can inspect it, + // unless explicitly disabled for benchmarking. + if (!ledgerCloseMeta && !mApp.getConfig().DISABLE_TX_META_FOR_TESTING) { ledgerCloseMeta = std::make_unique( header.current().ledgerVersion); @@ -1628,8 +1653,17 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, #endif { // first, prefetch source accounts for txset, then charge fees +#ifdef BUILD_TESTS + phaseStart = std::chrono::steady_clock::now(); +#endif prefetchTxSourceIds(mApp.getLedgerTxnRoot(), *applicableTxSet, mApp.getConfig()); +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.prefetchSourceAccountsMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif // Time the entire transaction processing phase from fee processing // through transaction application @@ -1638,10 +1672,26 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, // Subtle: after this call, `header` is invalidated, and is not safe // to use +#ifdef BUILD_TESTS + phaseStart = std::chrono::steady_clock::now(); +#endif auto const mutableTxResults = processFeesSeqNums( *applicableTxSet, ltx, ledgerCloseMeta, ledgerData); +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.processFeesSeqNumsMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); + phaseStart = std::chrono::steady_clock::now(); +#endif txResultSet = applyTransactions(*applicableTxSet, mutableTxResults, ltx, ledgerCloseMeta); +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyTransactionsMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif } if (mApp.getConfig().MODE_STORES_HISTORY_MISC) @@ -1660,6 +1710,9 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, mApplyState.markStartOfCommitting(); JITTER_INJECT_DELAY(); +#ifdef BUILD_TESTS + phaseStart = std::chrono::steady_clock::now(); +#endif bool upgradeApplied = false; for (size_t i = 0; i < sv.upgrades.size(); i++) { @@ -1710,13 +1763,28 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, CLOG_ERROR(Ledger, "Unknown exception during upgrade"); } } +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyUpgradesMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif auto maybeNewVersion = ltx.loadHeader().current().ledgerVersion; auto ledgerSeq = ltx.loadHeader().current().ledgerSeq; +#ifdef BUILD_TESTS + phaseStart = std::chrono::steady_clock::now(); +#endif auto lclSnap = mApplyState.copyLedgerStateSnapshot(); auto appliedLedgerState = sealLedgerTxnAndStoreInBucketsAndDB( lclSnap, ltx, ledgerCloseMeta, initialLedgerVers); +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sealAndBucketMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif // NB: from now on, the ledger state may not change, but LCL still hasn't // advanced properly. Hence when requesting the ledger state data (such as @@ -1823,10 +1891,20 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, JITTER_INJECT_DELAY(); // step 2 +#ifdef BUILD_TESTS + phaseStart = std::chrono::steady_clock::now(); +#endif ltx.commit(); +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sqlCommitMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); + phaseStart = std::chrono::steady_clock::now(); +#endif #ifdef BUILD_TESTS - mLatestTxResultSet = txResultSet; + mLatestTxResultSet = std::move(txResultSet); #endif // step 3 @@ -1880,6 +1958,12 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, }; mApp.postOnMainThread(std::move(cb), "advanceLedgerStateAndPublish"); } +#ifdef BUILD_TESTS + phaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.postCommitMs = + std::chrono::duration(phaseEnd - phaseStart) + .count(); +#endif maybeSimulateSleep(mApp.getConfig(), txSet->sizeOpTotalForLogging(), applyLedgerTime); @@ -2230,6 +2314,11 @@ LedgerManagerImpl::processFeesSeqNums( { LedgerTxn ltx(ltxOuter); auto header = ltx.loadHeader().current(); + // Cache protocol version to avoid repeated loadHeader() calls + // in the per-TX loop below. + auto const cachedLedgerVersion = header.ledgerVersion; + bool const isV19OrLater = protocolVersionStartsFrom( + cachedLedgerVersion, ProtocolVersion::V_19); std::map accToMaxSeq; #ifdef BUILD_TESTS @@ -2252,52 +2341,67 @@ LedgerManagerImpl::processFeesSeqNums( { for (auto const& tx : phase) { - LedgerTxn ltxTx(ltx); - txResults.push_back( - tx->processFeeSeqNum(ltxTx, txSet.getTxBaseFee(tx))); + // Common per-tx fee processing logic, parameterized on the + // active LTX (either a child for meta tracking, or the + // parent directly when meta is disabled). + auto processOneTxFee = [&](AbstractLedgerTxn& activeLtx) { + txResults.push_back(tx->processFeeSeqNum( + activeLtx, txSet.getTxBaseFee(tx))); #ifdef BUILD_TESTS - if (expectedResultsIter) - { - releaseAssert(*expectedResultsIter != - expectedResults->results.end()); - releaseAssert((*expectedResultsIter)->transactionHash == - tx->getContentsHash()); - txResults.back()->setReplayTransactionResult( - (*expectedResultsIter)->result); - - ++(*expectedResultsIter); - } -#endif // BUILD_TESTS - - if (protocolVersionStartsFrom( - ltxTx.loadHeader().current().ledgerVersion, - ProtocolVersion::V_19)) - { - auto res = - accToMaxSeq.emplace(tx->getSourceID(), tx->getSeqNum()); - if (!res.second) + if (expectedResultsIter) { - res.first->second = - std::max(res.first->second, tx->getSeqNum()); + releaseAssert(*expectedResultsIter != + expectedResults->results.end()); + releaseAssert((*expectedResultsIter)->transactionHash == + tx->getContentsHash()); + txResults.back()->setReplayTransactionResult( + (*expectedResultsIter)->result); + + ++(*expectedResultsIter); } +#endif // BUILD_TESTS - if (mergeOpInTx(tx->getRawOperations())) + // Merge-op tracking (accToMaxSeq) is only needed for + // non-Soroban TXs. Soroban TXs have exactly one + // InvokeHostFunction op and can never contain + // ACCOUNT_MERGE, so mergeSeen will never be set. + // Use cached version to avoid per-TX loadHeader() calls. + if (isV19OrLater && !tx->isSoroban()) { - mergeSeen = true; + auto res = accToMaxSeq.emplace(tx->getSourceID(), + tx->getSeqNum()); + if (!res.second) + { + res.first->second = + std::max(res.first->second, tx->getSeqNum()); + } + + if (mergeOpInTx(tx->getRawOperations())) + { + mergeSeen = true; + } } - } + }; if (ledgerCloseMeta) { + // Use a child LTX so we can capture per-tx changes + // for meta tracking via getChanges(). + LedgerTxn ltxTx(ltx); + processOneTxFee(ltxTx); ledgerCloseMeta->pushTxFeeProcessing(ltxTx.getChanges()); + ltxTx.commit(); + } + else + { + // No meta needed — operate directly on parent LTX to + // avoid per-tx child LTX creation/destruction overhead. + processOneTxFee(ltx); } ++index; - ltxTx.commit(); } } - if (protocolVersionStartsFrom(ltx.loadHeader().current().ledgerVersion, - ProtocolVersion::V_19) && - mergeSeen) + if (isV19OrLater && mergeSeen) { for (auto const& [accountID, seqNum] : accToMaxSeq) { @@ -2475,10 +2579,13 @@ LedgerManagerImpl::checkAllTxBundleInvariants( AppConnector& app, ApplyStage const& stage, Config const& config, ParallelLedgerInfo const& ledgerInfo, LedgerHeader const& header) { + bool const hasInvariants = !config.INVARIANT_CHECKS.empty(); for (auto const& txBundle : stage) { - // First check the invariants - if (txBundle.getResPayload().isSuccess()) + // Only run invariant checks if any invariants are enabled. + // The delta is not built when invariants are disabled (see + // parallelApply), so we must not call getDelta() in that case. + if (hasInvariants && txBundle.getResPayload().isSuccess()) { try { @@ -2506,7 +2613,6 @@ LedgerManagerImpl::checkAllTxBundleInvariants( // We don't call processPostApply for post v23 transactions at the // moment because processPostApply is currently a no-op for those - // transactions. txBundle.getEffects().getMeta().maybeSetRefundableFeeMeta( txBundle.getResPayload().getRefundableFeeTracker()); @@ -2523,12 +2629,44 @@ LedgerManagerImpl::applySorobanStage( auto const& config = app.getConfig(); auto ledgerInfo = getParallelLedgerInfo(app, header); +#ifdef BUILD_TESTS + auto subStart = std::chrono::steady_clock::now(); +#endif auto threadStates = applySorobanStageClustersInParallel( app, stage, globalParState, sorobanBasePrngSeed, config, ledgerInfo); +#ifdef BUILD_TESTS + auto subEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanParallelApplyMs += + std::chrono::duration(subEnd - subStart).count(); +#endif +#ifdef BUILD_TESTS + subStart = std::chrono::steady_clock::now(); +#endif checkAllTxBundleInvariants(app, stage, config, ledgerInfo, header); +#ifdef BUILD_TESTS + subEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanCheckInvariantsMs += + std::chrono::duration(subEnd - subStart).count(); +#endif +#ifdef BUILD_TESTS + subStart = std::chrono::steady_clock::now(); +#endif globalParState.commitChangesFromThreads(app, threadStates, stage); +#ifdef BUILD_TESTS + subEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanCommitFromThreadsMs += + std::chrono::duration(subEnd - subStart).count(); + + subStart = std::chrono::steady_clock::now(); +#endif + threadStates.clear(); +#ifdef BUILD_TESTS + subEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanDestroyThreadStatesMs += + std::chrono::duration(subEnd - subStart).count(); +#endif } void @@ -2538,18 +2676,51 @@ LedgerManagerImpl::applySorobanStages(AppConnector& app, AbstractLedgerTxn& ltx, Hash const& sorobanBasePrngSeed) { ZoneScoped; - GlobalParallelApplyLedgerState globalParState( - app, mApplyState.copyLedgerStateSnapshot(), ltx, stages, - mApplyState.getInMemorySorobanState(), sorobanConfig); - // LedgerTxn is not passed into applySorobanStage, so there's no risk - // of the header being updated while we apply the stages. - auto const& header = ltx.loadHeader().current(); - for (auto const& stage : stages) +#ifdef BUILD_TESTS + auto globalStart = std::chrono::steady_clock::now(); +#endif { - applySorobanStage(app, header, globalParState, stage, - sorobanBasePrngSeed); - } - globalParState.commitChangesToLedgerTxn(ltx); + GlobalParallelApplyLedgerState globalParState( + app, mApplyState.copyLedgerStateSnapshot(), ltx, stages, + mApplyState.getInMemorySorobanState(), sorobanConfig); +#ifdef BUILD_TESTS + auto globalEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanSetupGlobalMs = + std::chrono::duration(globalEnd - globalStart) + .count(); +#endif + // LedgerTxn is not passed into applySorobanStage, so there's no risk + // of the header being updated while we apply the stages. + auto const& header = ltx.loadHeader().current(); +#ifdef BUILD_TESTS + mLastPhaseTimings.sorobanParallelApplyMs = 0; + mLastPhaseTimings.sorobanCheckInvariantsMs = 0; + mLastPhaseTimings.sorobanCommitFromThreadsMs = 0; + mLastPhaseTimings.sorobanDestroyThreadStatesMs = 0; +#endif + for (auto const& stage : stages) + { + applySorobanStage(app, header, globalParState, stage, + sorobanBasePrngSeed); + } +#ifdef BUILD_TESTS + auto subStart = std::chrono::steady_clock::now(); +#endif + globalParState.commitChangesToLedgerTxn(ltx); +#ifdef BUILD_TESTS + auto subEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanCommitToLtxMs = + std::chrono::duration(subEnd - subStart) + .count(); + globalStart = std::chrono::steady_clock::now(); +#endif + } // globalParState destroyed here +#ifdef BUILD_TESTS + auto globalEnd2 = std::chrono::steady_clock::now(); + mLastPhaseTimings.sorobanDestroyGlobalStateMs = + std::chrono::duration(globalEnd2 - globalStart) + .count(); +#endif } void @@ -2589,7 +2760,10 @@ LedgerManagerImpl::processResultAndMeta( { auto metaXDR = txMetaBuilder.finalize(result.isSuccess()); #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back(metaXDR); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back(metaXDR); + } #endif ledgerCloseMeta->setTxProcessingMetaAndResultPair( @@ -2598,8 +2772,11 @@ LedgerManagerImpl::processResultAndMeta( else { #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back( - txMetaBuilder.finalize(result.isSuccess())); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back( + txMetaBuilder.finalize(result.isSuccess())); + } #endif } } @@ -2612,6 +2789,9 @@ LedgerManagerImpl::applyTransactions( std::unique_ptr const& ledgerCloseMeta) { ZoneNamedN(txsZone, "applyTransactions", true); +#ifdef BUILD_TESTS + auto txSubStart = std::chrono::steady_clock::now(); +#endif size_t numTxs = txSet.sizeTxTotal(); size_t numOps = txSet.sizeOpTotal(); releaseAssert(numTxs == mutableTxResults.size()); @@ -2633,7 +2813,21 @@ LedgerManagerImpl::applyTransactions( TransactionResultSet txResultSet; txResultSet.results.reserve(numTxs); +#ifdef BUILD_TESTS + auto txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyTxSetupMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); + txSubStart = std::chrono::steady_clock::now(); +#endif prefetchTransactionData(mApp.getLedgerTxnRoot(), txSet, mApp.getConfig()); +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.prefetchTxDataMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); + txSubStart = std::chrono::steady_clock::now(); +#endif auto phases = txSet.getPhasesInApplyOrder(); Hash sorobanBasePrngSeed = txSet.getContentsHash(); @@ -2645,8 +2839,20 @@ LedgerManagerImpl::applyTransactions( bool enableTxMeta = ledgerCloseMeta != nullptr; #ifdef BUILD_TESTS // In tests we want to always enable tx meta because we store it in - // mLastLedgerTxMeta. - enableTxMeta = true; + // mLastLedgerTxMeta, unless explicitly disabled for benchmarking. + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + enableTxMeta = true; + } +#endif +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyTxMidSetupMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); +#endif +#ifdef BUILD_TESTS + txSubStart = std::chrono::steady_clock::now(); #endif std::optional sorobanConfig; if (protocolVersionStartsFrom(ltx.loadHeader().current().ledgerVersion, @@ -2655,6 +2861,13 @@ LedgerManagerImpl::applyTransactions( sorobanConfig = std::make_optional(SorobanNetworkConfig::loadFromLedger(ltx)); } +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.loadSorobanConfigMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); + mLastPhaseTimings.applySeqClassicMs = 0; +#endif std::vector applyStages; for (auto const& phase : phases) { @@ -2663,9 +2876,19 @@ LedgerManagerImpl::applyTransactions( try { releaseAssert(sorobanConfig.has_value()); +#ifdef BUILD_TESTS + auto parPhaseStart = std::chrono::steady_clock::now(); +#endif applyParallelPhase(phase, applyStages, mutableTxResults, index, ltx, enableTxMeta, *sorobanConfig, sorobanBasePrngSeed); +#ifdef BUILD_TESTS + auto parPhaseEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyParallelPhaseTotalMs = + std::chrono::duration(parPhaseEnd - + parPhaseStart) + .count(); +#endif } catch (std::exception const& e) { @@ -2680,15 +2903,34 @@ LedgerManagerImpl::applyTransactions( } else { +#ifdef BUILD_TESTS + txSubStart = std::chrono::steady_clock::now(); +#endif applySequentialPhase(phase, mutableTxResults, index, ltx, enableTxMeta, sorobanConfig, sorobanBasePrngSeed, ledgerCloseMeta, txResultSet); +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applySeqClassicMs += + std::chrono::duration(txSubEnd - txSubStart) + .count(); +#endif } } +#ifdef BUILD_TESTS + txSubStart = std::chrono::steady_clock::now(); +#endif processPostTxSetApply(phases, applyStages, ltx, ledgerCloseMeta, txResultSet); +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.postTxSetApplyMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); + txSubStart = std::chrono::steady_clock::now(); +#endif // Update cluster and stage metrics if (!applyStages.empty()) @@ -2703,6 +2945,21 @@ LedgerManagerImpl::applyTransactions( } logTxApplyMetrics(ltx, numTxs, numOps); +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.applyTxTailMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); + + txSubStart = std::chrono::steady_clock::now(); +#endif + applyStages.clear(); +#ifdef BUILD_TESTS + txSubEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.destroyApplyStagesMs = + std::chrono::duration(txSubEnd - txSubStart) + .count(); +#endif return txResultSet; } @@ -2719,6 +2976,9 @@ LedgerManagerImpl::applyParallelPhase( applyStages.reserve(txSetStages.size()); +#ifdef BUILD_TESTS + auto bundleStart = std::chrono::steady_clock::now(); +#endif for (auto const& stage : txSetStages) { std::vector applyClusters; @@ -2758,6 +3018,12 @@ LedgerManagerImpl::applyParallelPhase( } applyStages.emplace_back(std::move(applyClusters)); } +#ifdef BUILD_TESTS + auto bundleEnd = std::chrono::steady_clock::now(); + mLastPhaseTimings.buildTxBundlesMs = + std::chrono::duration(bundleEnd - bundleStart) + .count(); +#endif applySorobanStages(mApp.getAppConnector(), ltx, applyStages, sorobanConfig, sorobanBasePrngSeed); @@ -2840,7 +3106,9 @@ LedgerManagerImpl::processPostTxSetApply( { for (auto const& txBundle : stage) { + if (ledgerCloseMeta) { + // Use child LTX for meta change tracking. LedgerTxn ltxInner(ltx); txBundle.getTx()->processPostTxSetApply( mApp.getAppConnector(), ltxInner, @@ -2849,13 +3117,20 @@ LedgerManagerImpl::processPostTxSetApply( .getMeta() .getTxEventManager()); - if (ledgerCloseMeta) - { - ledgerCloseMeta->setPostTxApplyFeeProcessing( - ltxInner.getChanges(), txBundle.getTxNum()); - } + ledgerCloseMeta->setPostTxApplyFeeProcessing( + ltxInner.getChanges(), txBundle.getTxNum()); ltxInner.commit(); } + else + { + // No meta — operate directly on parent LTX. + txBundle.getTx()->processPostTxSetApply( + mApp.getAppConnector(), ltx, + txBundle.getResPayload(), + txBundle.getEffects() + .getMeta() + .getTxEventManager()); + } // setPostTxApplyFeeProcessing can update the feeCharged in // the result, so this needs to be done after @@ -2948,18 +3223,24 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( // `ledgerApplied` protects this call with a mutex std::vector initEntries, liveEntries; std::vector deadEntries; + + // Future for async hot archive batch operation. + // addHotArchiveBatch modifies mHotArchiveBucketList which is independent + // from mLiveBucketList (modified by addLiveBatch). + std::future hotArchiveBatchFuture; + // Any V20 features must be behind initialLedgerVers check, see comment // in LedgerManagerImpl::ledgerApplied if (protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION)) { - // In `getAllTTLKeysWithoutSealing` it is important not to seal ltx, - // because it is still being modified by the eviction flow. - // `getAllTTLKeysWithoutSealing` must be called at the right time - // _after_ all operations have been applied, but _before_ evictions. - auto sorobanConfig = SorobanNetworkConfig::loadFromLedger(ltx); + // resolveBackgroundEvictionScan checks modified keys via direct O(1) + // lookups in the LedgerTxn's EntryMap (isModifiedKey), avoiding the + // need to build a full UnorderedSet of all modified keys. + // It must be called at the right time _after_ all operations have + // been applied, but _before_ evictions (ltx must not be sealed). auto evictedState = - mApp.getBucketManager().resolveBackgroundEvictionScan( - lclSnapshot, ltx, ltx.getAllKeysWithoutSealing()); + mApp.getBucketManager().resolveBackgroundEvictionScan(lclSnapshot, + ltx); if (protocolVersionStartsFrom( initialLedgerVers, @@ -2996,9 +3277,20 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } else { - mApp.getBucketManager().addHotArchiveBatch( - mApp, lh, evictedState.archivedEntries, - restoredHotArchiveKeys); + // Launch addHotArchiveBatch asynchronously. It modifies + // mHotArchiveBucketList which is independent from + // mLiveBucketList, so it can run in parallel with addLiveBatch. + auto& bucketManager = mApp.getBucketManager(); + auto archivedEntries = evictedState.archivedEntries; + hotArchiveBatchFuture = + std::async(std::launch::async, [&bucketManager, this, lh, + archivedEntries, + restoredHotArchiveKeys]() { + ZoneScopedN("addHotArchiveBatch (async)"); + bucketManager.addHotArchiveBatch( + mApp, lh, archivedEntries, restoredHotArchiveKeys); + }); + // Validate evicted entries against Protocol 23 corruption // data if configured if (mApp.getProtocol23CorruptionDataVerifier()) @@ -3038,12 +3330,40 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } // NB: getAllEntries seals the ltx. ltx.getAllEntries(initEntries, liveEntries, deadEntries); + + // Launch async task to update in-memory Soroban state. This is independent + // from both addHotArchiveBatch and addLiveBatch: + // - addHotArchiveBatch modifies mHotArchiveBucketList + // - addLiveBatch modifies mLiveBucketList + // - updateState modifies mInMemorySorobanState + // All three can run in parallel. + std::future inMemoryStateUpdateFuture; + + auto& inMemoryState = mApplyState.getInMemorySorobanStateForUpdate(); + auto& sorobanMetrics = mApplyState.getMetrics().mSorobanMetrics; + + inMemoryStateUpdateFuture = std::async( + std::launch::async, + [&inMemoryState, &initEntries, &liveEntries, &deadEntries, &lh, + &finalSorobanConfig, &sorobanMetrics]() { + ZoneScopedN("updateInMemorySorobanState (async)"); + inMemoryState.updateState(initEntries, liveEntries, deadEntries, lh, + finalSorobanConfig, sorobanMetrics); + }); + mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, initEntries); mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, liveEntries); mApp.getBucketManager().addLiveBatch(mApp, lh, initEntries, liveEntries, deadEntries); - mApplyState.updateInMemorySorobanState(initEntries, liveEntries, - deadEntries, lh, finalSorobanConfig); + // Wait for all async operations to complete before returning. + if (hotArchiveBatchFuture.valid()) + { + hotArchiveBatchFuture.get(); + } + if (inMemoryStateUpdateFuture.valid()) + { + inMemoryStateUpdateFuture.get(); + } return finalSorobanConfig; } diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index e5350f826a..e6b7c8ee1b 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -228,6 +228,10 @@ class LedgerManagerImpl : public LedgerManager std::vector const& deadEntries, LedgerHeader const& lh, std::optional const& sorobanConfig); + // Returns mutable reference to in-memory state for direct updates. + // Only safe during COMMITTING phase when no readers are active. + InMemorySorobanState& getInMemorySorobanStateForUpdate(); + // Note: These are const getters, but should still only be called in the // COMMITTING phase. uint64_t getSorobanInMemoryStateSize() const; @@ -515,6 +519,37 @@ class LedgerManagerImpl : public LedgerManager std::chrono::milliseconds getExpectedLedgerCloseTime() const override; #ifdef BUILD_TESTS + struct LedgerClosePhaseTimings + { + double prepareTxSetMs = 0; + double prefetchSourceAccountsMs = 0; + double processFeesSeqNumsMs = 0; + double applyTransactionsMs = 0; + double applyTxSetupMs = 0; + double prefetchTxDataMs = 0; + double applyTxMidSetupMs = 0; + double loadSorobanConfigMs = 0; + double buildTxBundlesMs = 0; + double sorobanSetupGlobalMs = 0; + double sorobanParallelApplyMs = 0; + double sorobanCheckInvariantsMs = 0; + double sorobanCommitFromThreadsMs = 0; + double sorobanDestroyThreadStatesMs = 0; + double sorobanCommitToLtxMs = 0; + double sorobanDestroyGlobalStateMs = 0; + double applyParallelPhaseTotalMs = 0; + double applySeqClassicMs = 0; + double postTxSetApplyMs = 0; + double applyTxTailMs = 0; + double destroyApplyStagesMs = 0; + double applyUpgradesMs = 0; + double sealAndBucketMs = 0; + double sqlCommitMs = 0; + double postCommitMs = 0; + }; + + LedgerClosePhaseTimings const& getLastPhaseTimings() const; + std::vector const& getLastClosedLedgerTxMeta() override; std::optional const& @@ -527,6 +562,8 @@ class LedgerManagerImpl : public LedgerManager getModuleCacheForTesting() override; void rebuildInMemorySorobanStateForTesting(uint32_t ledgerVersion) override; uint64_t getSorobanInMemoryStateSizeForTesting() override; + + LedgerClosePhaseTimings mLastPhaseTimings; #endif uint64_t secondsSinceLastLedgerClose() const override; diff --git a/src/ledger/LedgerTxn.cpp b/src/ledger/LedgerTxn.cpp index 2e4df90ce4..1fde30f7eb 100644 --- a/src/ledger/LedgerTxn.cpp +++ b/src/ledger/LedgerTxn.cpp @@ -409,6 +409,22 @@ AbstractLedgerTxn::~AbstractLedgerTxn() { } +void +AbstractLedgerTxn::createWithoutLoading(InternalLedgerEntry&& entry) +{ + // Default: forward to const-ref version (copies). + // LedgerTxn overrides this to move directly into make_shared. + createWithoutLoading(static_cast(entry)); +} + +void +AbstractLedgerTxn::updateWithoutLoading(InternalLedgerEntry&& entry) +{ + // Default: forward to const-ref version (copies). + // LedgerTxn overrides this to move directly into make_shared. + updateWithoutLoading(static_cast(entry)); +} + // Implementation of LedgerTxn ---------------------------------------------- LedgerTxn::LedgerTxn(AbstractLedgerTxnParent& parent, bool shouldUpdateLastModified, TransactionMode mode) @@ -770,6 +786,32 @@ LedgerTxn::Impl::createWithoutLoading(InternalLedgerEntry const& entry) /* effectiveActive */ false); } +void +LedgerTxn::createWithoutLoading(InternalLedgerEntry&& entry) +{ + getImpl()->createWithoutLoading(std::move(entry)); +} + +void +LedgerTxn::Impl::createWithoutLoading(InternalLedgerEntry&& entry) +{ + abortIfWrongThread("createWithoutLoading"); + throwIfSealed(); + throwIfChild(); + + auto key = entry.toKey(); + auto iter = mActive.find(key); + if (iter != mActive.end()) + { + throw std::runtime_error("Key is already active"); + } + + updateEntry(key, /* keyHint */ nullptr, + LedgerEntryPtr::Init( + std::make_shared(std::move(entry))), + /* effectiveActive */ false); +} + void LedgerTxn::updateWithoutLoading(InternalLedgerEntry const& entry) { @@ -796,6 +838,32 @@ LedgerTxn::Impl::updateWithoutLoading(InternalLedgerEntry const& entry) /* effectiveActive */ false); } +void +LedgerTxn::updateWithoutLoading(InternalLedgerEntry&& entry) +{ + getImpl()->updateWithoutLoading(std::move(entry)); +} + +void +LedgerTxn::Impl::updateWithoutLoading(InternalLedgerEntry&& entry) +{ + abortIfWrongThread("updateWithoutLoading"); + throwIfSealed(); + throwIfChild(); + + auto key = entry.toKey(); + auto iter = mActive.find(key); + if (iter != mActive.end()) + { + throw std::runtime_error("Key is already active"); + } + + updateEntry(key, /* keyHint */ nullptr, + LedgerEntryPtr::Live( + std::make_shared(std::move(entry))), + /* effectiveActive */ false); +} + void LedgerTxn::deactivate(InternalLedgerKey const& key) { @@ -1628,6 +1696,7 @@ LedgerTxn::Impl::getAllEntries(std::vector& initEntries, std::vector& liveEntries, std::vector& deadEntries) { + ZoneScoped; abortIfWrongThread("getAllEntries"); std::vector resInit, resLive; std::vector resDead; @@ -1695,30 +1764,17 @@ LedgerTxn::Impl::getRestoredLiveBucketListKeys() const return mRestoredEntries.liveBucketList; } -LedgerKeySet -LedgerTxn::getAllKeysWithoutSealing() const +bool +LedgerTxn::isModifiedKey(LedgerKey const& key) const { - return getImpl()->getAllKeysWithoutSealing(); + return getImpl()->isModifiedKey(key); } -LedgerKeySet -LedgerTxn::Impl::getAllKeysWithoutSealing() const +bool +LedgerTxn::Impl::isModifiedKey(LedgerKey const& key) const { - abortIfWrongThread("getAllKeysWithoutSealing"); - throwIfNotExactConsistency(); - LedgerKeySet result; - // Subtle: mEntry contains only *modified* entries in this LedgerTxn. - // Callers rely on this — for example, to enforce that expired entries - // (which cannot be modified) are never present here. - for (auto const& [k, v] : mEntry) - { - if (k.type() == InternalLedgerEntryType::LEDGER_ENTRY) - { - result.emplace(k.ledgerKey()); - } - } - - return result; + abortIfWrongThread("isModifiedKey"); + return mEntry.find(InternalLedgerKey(key)) != mEntry.end(); } std::shared_ptr diff --git a/src/ledger/LedgerTxn.h b/src/ledger/LedgerTxn.h index b9decf389b..bddfc7f6d4 100644 --- a/src/ledger/LedgerTxn.h +++ b/src/ledger/LedgerTxn.h @@ -651,6 +651,12 @@ class AbstractLedgerTxn : public AbstractLedgerTxnParent virtual void createWithoutLoading(InternalLedgerEntry const& entry) = 0; virtual void updateWithoutLoading(InternalLedgerEntry const& entry) = 0; + // Move overloads: avoid deep-copying InternalLedgerEntry when the caller + // is consuming a temporary or explicitly moving ownership. Default + // implementations forward to the const& versions; LedgerTxn overrides + // to move directly into make_shared for zero-copy insertion. + virtual void createWithoutLoading(InternalLedgerEntry&& entry); + virtual void updateWithoutLoading(InternalLedgerEntry&& entry); virtual void eraseWithoutLoading(InternalLedgerKey const& key) = 0; // getChanges, getDelta, and getAllEntries are used to @@ -678,10 +684,10 @@ class AbstractLedgerTxn : public AbstractLedgerTxnParent std::vector& liveEntries, std::vector& deadEntries) = 0; - // Returns all TTL keys that have been modified (create, update, and - // delete), but does not cause the AbstractLedgerTxn or update last - // modified. - virtual LedgerKeySet getAllKeysWithoutSealing() const = 0; + // Returns true if the given LedgerKey has been modified (created, updated, + // or deleted) in this LedgerTxn. This is an O(1) lookup that avoids + // building the full key set. + virtual bool isModifiedKey(LedgerKey const& key) const = 0; // forAllWorstBestOffers allows a parent AbstractLedgerTxn to process the // worst best offers (an offer is a worst best offer if every better offer @@ -817,7 +823,7 @@ class LedgerTxn : public AbstractLedgerTxn void getAllEntries(std::vector& initEntries, std::vector& liveEntries, std::vector& deadEntries) override; - LedgerKeySet getAllKeysWithoutSealing() const override; + bool isModifiedKey(LedgerKey const& key) const override; UnorderedMap getRestoredHotArchiveKeys() const override; @@ -834,6 +840,8 @@ class LedgerTxn : public AbstractLedgerTxn void createWithoutLoading(InternalLedgerEntry const& entry) override; void updateWithoutLoading(InternalLedgerEntry const& entry) override; + void createWithoutLoading(InternalLedgerEntry&& entry) override; + void updateWithoutLoading(InternalLedgerEntry&& entry) override; void eraseWithoutLoading(InternalLedgerKey const& key) override; std::map> loadAllOffers() override; diff --git a/src/ledger/LedgerTxnImpl.h b/src/ledger/LedgerTxnImpl.h index 95f46b042b..848bfde89f 100644 --- a/src/ledger/LedgerTxnImpl.h +++ b/src/ledger/LedgerTxnImpl.h @@ -436,7 +436,7 @@ class LedgerTxn::Impl UnorderedMap getRestoredHotArchiveKeys() const; UnorderedMap getRestoredLiveBucketListKeys() const; - LedgerKeySet getAllKeysWithoutSealing() const; + bool isModifiedKey(LedgerKey const& key) const; // getNewestVersion has the basic exception safety guarantee. If it throws // an exception, then @@ -458,10 +458,12 @@ class LedgerTxn::Impl // createWithoutLoading has the strong exception safety guarantee. // If it throws an exception, then the current LedgerTxn::Impl is unchanged. void createWithoutLoading(InternalLedgerEntry const& entry); + void createWithoutLoading(InternalLedgerEntry&& entry); // updateWithoutLoading has the strong exception safety guarantee. // If it throws an exception, then the current LedgerTxn::Impl is unchanged. void updateWithoutLoading(InternalLedgerEntry const& entry); + void updateWithoutLoading(InternalLedgerEntry&& entry); // eraseWithoutLoading has the strong exception safety guarantee. If it // throws an exception, then the current LedgerTxn::Impl is unchanged. diff --git a/src/ledger/test/InMemoryLedgerTxn.cpp b/src/ledger/test/InMemoryLedgerTxn.cpp index d95e7733f4..7d881b52f0 100644 --- a/src/ledger/test/InMemoryLedgerTxn.cpp +++ b/src/ledger/test/InMemoryLedgerTxn.cpp @@ -248,6 +248,14 @@ InMemoryLedgerTxn::createWithoutLoading(InternalLedgerEntry const& entry) updateLedgerKeyMap(entry.toKey(), true); } +void +InMemoryLedgerTxn::createWithoutLoading(InternalLedgerEntry&& entry) +{ + auto key = entry.toKey(); + LedgerTxn::createWithoutLoading(std::move(entry)); + updateLedgerKeyMap(key, true); +} + void InMemoryLedgerTxn::updateWithoutLoading(InternalLedgerEntry const& entry) { @@ -255,6 +263,14 @@ InMemoryLedgerTxn::updateWithoutLoading(InternalLedgerEntry const& entry) updateLedgerKeyMap(entry.toKey(), true); } +void +InMemoryLedgerTxn::updateWithoutLoading(InternalLedgerEntry&& entry) +{ + auto key = entry.toKey(); + LedgerTxn::updateWithoutLoading(std::move(entry)); + updateLedgerKeyMap(key, true); +} + void InMemoryLedgerTxn::eraseWithoutLoading(InternalLedgerKey const& key) { diff --git a/src/ledger/test/InMemoryLedgerTxn.h b/src/ledger/test/InMemoryLedgerTxn.h index ab0c501f89..100dfdb486 100644 --- a/src/ledger/test/InMemoryLedgerTxn.h +++ b/src/ledger/test/InMemoryLedgerTxn.h @@ -107,7 +107,9 @@ class InMemoryLedgerTxn : public LedgerTxn void rollbackChild() noexcept override; void createWithoutLoading(InternalLedgerEntry const& entry) override; + void createWithoutLoading(InternalLedgerEntry&& entry) override; void updateWithoutLoading(InternalLedgerEntry const& entry) override; + void updateWithoutLoading(InternalLedgerEntry&& entry) override; void eraseWithoutLoading(InternalLedgerKey const& key) override; LedgerTxnEntry create(InternalLedgerEntry const& entry) override; diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 13abb8a517..f0876482cb 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,14 @@ static std::unordered_set const TESTING_SUGGESTED_OPTIONS = { namespace { +int +defaultLedgerCloseWorkerThreads() +{ + auto const hardwareThreads = + static_cast(std::thread::hardware_concurrency()); + return std::max(1, hardwareThreads - 2); +} + // compute a default threshold for qset: // if thresholdLevel is SIMPLE_MAJORITY there are no inner sets, only // require majority @@ -172,6 +181,7 @@ Config::Config() : NODE_SEED(SecretKey::random()) BACKGROUND_OVERLAY_PROCESSING = true; PARALLEL_LEDGER_APPLY = true; DISABLE_SOROBAN_METRICS_FOR_TESTING = false; + DISABLE_TX_META_FOR_TESTING = false; BACKGROUND_TX_SIG_VERIFICATION = true; BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 14; // 2^14 == 16 kb BUCKETLIST_DB_INDEX_CUTOFF = 20; // 20 mb @@ -289,6 +299,10 @@ Config::Config() : NODE_SEED(SecretKey::random()) // Worst case = 10 concurrent merges + 1 quorum intersection calculation. WORKER_THREADS = 11; + // Leave headroom for the main thread and one additional thread while still + // scaling ledger close parallelism with the host. + LEDGER_CLOSE_WORKER_THREADS = defaultLedgerCloseWorkerThreads(); + // Compilation is a short process that runs at startup and is CPU limited. // Empirically it tends to peak and start getting slower around 6 threads // due to coordination overhead between the producer and consumer threads. @@ -1180,6 +1194,8 @@ Config::processConfig(std::shared_ptr t) [&]() { DISABLE_SOROBAN_METRICS_FOR_TESTING = readBool(item); }}, + {"DISABLE_TX_META_FOR_TESTING", + [&]() { DISABLE_TX_META_FOR_TESTING = readBool(item); }}, {"EXPERIMENTAL_BACKGROUND_TX_SIG_VERIFICATION", [&]() { CLOG_WARNING(Overlay, @@ -1454,6 +1470,10 @@ Config::processConfig(std::shared_ptr t) [&]() { COMMANDS = readArray(item); }}, {"WORKER_THREADS", [&]() { WORKER_THREADS = readInt(item, 2, 1000); }}, + {"LEDGER_CLOSE_WORKER_THREADS", + [&]() { + LEDGER_CLOSE_WORKER_THREADS = readInt(item, 1, 100); + }}, {"QUERY_THREAD_POOL_SIZE", [&]() { QUERY_THREAD_POOL_SIZE = readInt(item, 1, 1000); diff --git a/src/main/Config.h b/src/main/Config.h index cb217d87c1..18b338a0b1 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -550,6 +550,11 @@ class Config : public std::enable_shared_from_this // Disable expensive Soroban metrics for performance testing bool DISABLE_SOROBAN_METRICS_FOR_TESTING; + // Disable transaction metadata collection in test builds for benchmarking. + // When true, BUILD_TESTS overrides that force ledgerCloseMeta allocation + // and enableTxMeta are suppressed, avoiding significant XDR copy overhead. + bool DISABLE_TX_META_FOR_TESTING; + // Batch transactions for flooding purposes (experimental). // Has no effect on non-test builds. size_t EXPERIMENTAL_TX_BATCH_MAX_SIZE; @@ -748,6 +753,9 @@ class Config : public std::enable_shared_from_this // thread-management config int WORKER_THREADS; + // Number of threads to use during ledger close parallelism + int LEDGER_CLOSE_WORKER_THREADS; + // Number of threads to serve query commands int QUERY_THREAD_POOL_SIZE; diff --git a/src/main/test/ConfigTests.cpp b/src/main/test/ConfigTests.cpp index db47c68fdf..a8e3ec8b81 100644 --- a/src/main/test/ConfigTests.cpp +++ b/src/main/test/ConfigTests.cpp @@ -793,7 +793,7 @@ TEST_CASE("secret resolution", "[config]") } stdfs::permissions(tmpPath, stdfs::perms::owner_read | stdfs::perms::owner_write); - auto otherKey = SecretKey::random().getStrKeyPublic(); + auto otherKey = SecretKey::pseudoRandomForTesting().getStrKeyPublic(); std::string configStr = R"( NODE_SEED="$FILE:)" + tmpPath + R"(" @@ -811,7 +811,7 @@ VALIDATORS=[")" + otherKey + R"( A"] SECTION("backward compatibility - inline NODE_SEED") { - auto otherKey = SecretKey::random().getStrKeyPublic(); + auto otherKey = SecretKey::pseudoRandomForTesting().getStrKeyPublic(); std::string configStr = R"( NODE_SEED=")" + testSeed + R"( self" UNSAFE_QUORUM=true @@ -834,7 +834,7 @@ VALIDATORS=[")" + otherKey + R"( A"] } stdfs::permissions(tmpPath, stdfs::perms::owner_read | stdfs::perms::owner_write); - auto otherKey = SecretKey::random().getStrKeyPublic(); + auto otherKey = SecretKey::pseudoRandomForTesting().getStrKeyPublic(); std::string configStr = R"( NODE_SEED="$FILE:)" + tmpPath + R"(" diff --git a/src/rust/CppShims.h b/src/rust/CppShims.h index 45114c2317..b16c0723bb 100644 --- a/src/rust/CppShims.h +++ b/src/rust/CppShims.h @@ -5,6 +5,10 @@ #pragma once #include "util/Logging.h" +#include +#include +#include +#include // This file just contains "shims" which are global C++ functions that cxx.rs // can understand how to call, that themselves call through to C++ code in some @@ -36,4 +40,16 @@ shim_logAtPartitionAndLevel(std::string const& partition, LogLevel level, { Logging::logAtPartitionAndLevel(partition, level, msg); } + +inline std::unique_ptr> +shim_copyU8Vector(std::uint8_t const* data, std::size_t len) +{ + auto copy = std::make_unique>(); + copy->reserve(len); + for (std::size_t i = 0; i < len; ++i) + { + copy->emplace_back(data[i]); + } + return copy; +} } diff --git a/src/rust/src/bridge.rs b/src/rust/src/bridge.rs index 87666bba75..f200ebca64 100644 --- a/src/rust/src/bridge.rs +++ b/src/rust/src/bridge.rs @@ -199,7 +199,7 @@ pub(crate) mod rust_bridge { restored_rw_entry_indices: &Vec, source_account: &CxxBuf, auth_entries: &Vec, - ledger_info: CxxLedgerInfo, + ledger_info: &CxxLedgerInfo, ledger_entries: &Vec, ttl_entries: &Vec, base_prng_seed: &CxxBuf, @@ -390,6 +390,7 @@ pub(crate) mod rust_bridge { level: LogLevel, msg: &CxxString, ) -> Result<()>; + unsafe fn shim_copyU8Vector(data: *const u8, len: usize) -> UniquePtr>; } } diff --git a/src/rust/src/common.rs b/src/rust/src/common.rs index cec25ddc8c..48f1247d98 100644 --- a/src/rust/src/common.rs +++ b/src/rust/src/common.rs @@ -1,5 +1,8 @@ use crate::{BridgeError, CxxBuf, RustBuf}; +#[cfg(feature = "testutils")] +use crate::CxxLedgerInfo; + impl From> for RustBuf { fn from(value: Vec) -> Self { Self { data: value } @@ -31,6 +34,41 @@ impl CxxBuf { } } +#[cfg(feature = "testutils")] +impl Clone for CxxBuf { + fn clone(&self) -> Self { + if self.data.is_null() { + return Self { + data: cxx::UniquePtr::null(), + }; + } + + let bytes = self.as_ref(); + Self { + data: unsafe { crate::rust_bridge::shim_copyU8Vector(bytes.as_ptr(), bytes.len()) }, + } + } +} + +#[cfg(feature = "testutils")] +impl Clone for CxxLedgerInfo { + fn clone(&self) -> Self { + Self { + protocol_version: self.protocol_version, + sequence_number: self.sequence_number, + timestamp: self.timestamp, + network_id: self.network_id.clone(), + base_reserve: self.base_reserve, + memory_limit: self.memory_limit, + min_temp_entry_ttl: self.min_temp_entry_ttl, + min_persistent_entry_ttl: self.min_persistent_entry_ttl, + max_entry_ttl: self.max_entry_ttl, + cpu_cost_params: self.cpu_cost_params.clone(), + mem_cost_params: self.mem_cost_params.clone(), + } + } +} + impl std::fmt::Display for BridgeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) diff --git a/src/rust/src/soroban_invoke.rs b/src/rust/src/soroban_invoke.rs index e9c5fc8c93..4ecbf753f5 100644 --- a/src/rust/src/soroban_invoke.rs +++ b/src/rust/src/soroban_invoke.rs @@ -13,63 +13,29 @@ pub(crate) fn invoke_host_function( restored_rw_entry_indices: &Vec, source_account_buf: &CxxBuf, auth_entries: &Vec, - ledger_info: CxxLedgerInfo, + ledger_info: &CxxLedgerInfo, ledger_entries: &Vec, ttl_entries: &Vec, base_prng_seed: &CxxBuf, rent_fee_configuration: CxxRentFeeConfiguration, module_cache: &SorobanModuleCache, ) -> Result> { - use std::error::Error as StdError; - type BoxStdErr = Box; - type BoxStdErrSend = Box; - type BoxStdErrSendSync = Box; - - fn sendable_str_err(str: &str) -> BoxStdErrSend { - let tmp: BoxStdErrSendSync = Box::from(str); - tmp as BoxStdErrSend - } - let hm = get_host_module_for_protocol(config_max_protocol, ledger_info.protocol_version)?; - // Rust stacks are 2MiB by default, which is a little too small - // for comfort; to give ourselves a little more breathing room - // against unforeseen bugs we use a 100MiB stack. Unfortunately - // there's no easy way to enforce this at the C++ side when the - // initial std::async parallel-exec thread is spawned, so we - // have to spawn _another_ here. On linux this is fairly fast, - // on the order of a ten-ish microseconds. - let LARGE_STACK_SIZE: usize = 100 * 1024 * 1024; // 100 MiB - let res = std::thread::scope(|scope| { - std::thread::Builder::new() - .stack_size(LARGE_STACK_SIZE) - .spawn_scoped(scope, || { - (hm.invoke_host_function)( - enable_diagnostics, - instruction_limit, - hf_buf, - &resources_buf, - restored_rw_entry_indices, - source_account_buf, - auth_entries, - &ledger_info, - ledger_entries, - ttl_entries, - base_prng_seed, - &rent_fee_configuration, - module_cache, - ) - // Map non-sendable error to sendable for crossing thread boundary. - // This is crude but the error is going to be stringified on the - // bridge-crossing anyways. - .map_err(|e| sendable_str_err(&format!("{e}"))) - }) - .map_err(|_| sendable_str_err("spawn_scoped failed"))? - .join() - .map_err(|_| sendable_str_err("join failed"))? - }); - - // Map sendable error back to non-sendable -- Rust doesn't do dyn upcasts. - let res = res.map_err(|e: BoxStdErrSend| e as BoxStdErr); + let res = (hm.invoke_host_function)( + enable_diagnostics, + instruction_limit, + hf_buf, + &resources_buf, + restored_rw_entry_indices, + source_account_buf, + auth_entries, + ledger_info, + ledger_entries, + ttl_entries, + base_prng_seed, + &rent_fee_configuration, + module_cache, + ); #[cfg(feature = "testutils")] crate::soroban_test_extra_protocol::maybe_invoke_host_function_again_and_compare_outputs( diff --git a/src/rust/src/soroban_test_extra_protocol.rs b/src/rust/src/soroban_test_extra_protocol.rs index ec5e8c787b..eb45b99909 100644 --- a/src/rust/src/soroban_test_extra_protocol.rs +++ b/src/rust/src/soroban_test_extra_protocol.rs @@ -25,7 +25,7 @@ pub(super) fn maybe_invoke_host_function_again_and_compare_outputs( restored_rw_entry_indices: &Vec, source_account_buf: &CxxBuf, auth_entries: &Vec, - mut ledger_info: CxxLedgerInfo, + ledger_info: &CxxLedgerInfo, ledger_entries: &Vec, ttl_entries: &Vec, base_prng_seed: &CxxBuf, @@ -36,6 +36,7 @@ pub(super) fn maybe_invoke_host_function_again_and_compare_outputs( if let Ok(proto) = u32::from_str(&extra) { info!(target: TX, "comparing soroban host for protocol {} with {}", ledger_info.protocol_version, proto); if let Ok(hm2) = get_host_module_for_protocol(proto, proto) { + let mut ledger_info = ledger_info.clone(); if let Err(e) = modify_ledger_info_for_extra_test_execution(&mut ledger_info, proto) { warn!(target: TX, "modifying ledger info for protocol {} re-execution failed: {:?}", proto, e); diff --git a/src/simulation/ApplyLoad.cpp b/src/simulation/ApplyLoad.cpp index 0508f7531f..c46ece9f41 100644 --- a/src/simulation/ApplyLoad.cpp +++ b/src/simulation/ApplyLoad.cpp @@ -13,6 +13,7 @@ #include "bucket/test/BucketTestUtils.h" #include "herder/Herder.h" #include "herder/HerderImpl.h" +#include "herder/TxSetFrame.h" #include "ledger/InMemorySorobanState.h" #include "ledger/LedgerManager.h" #include "ledger/LedgerManagerImpl.h" @@ -88,6 +89,291 @@ interpolatePercentile(std::vector const& sortedValues, return sortedValues[lo] * (1.0 - weight) + sortedValues[hi] * weight; } +struct PhaseStats +{ + double mean = 0; + double stddev = 0; + double p25 = 0; + double median = 0; + double p75 = 0; + double p95 = 0; + double p99 = 0; +}; + +PhaseStats +computePhaseStats(std::vector& values) +{ + PhaseStats s; + if (values.empty()) + { + return s; + } + double sum = std::accumulate(values.begin(), values.end(), 0.0); + s.mean = sum / values.size(); + double varianceSum = 0.0; + for (auto v : values) + { + double d = v - s.mean; + varianceSum += d * d; + } + s.stddev = std::sqrt(varianceSum / values.size()); + std::sort(values.begin(), values.end()); + s.p25 = interpolatePercentile(values, 25.0); + s.median = interpolatePercentile(values, 50.0); + s.p75 = interpolatePercentile(values, 75.0); + s.p95 = interpolatePercentile(values, 95.0); + s.p99 = interpolatePercentile(values, 99.0); + return s; +} + +void +logPhaseTimingsTable( + std::vector const& allTimings) +{ + if (allTimings.empty()) + { + return; + } + // Extract per-phase vectors. + size_t n = allTimings.size(); + + // Helper to extract a field into a vector. + auto extract = [&](auto field) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) + { + v[i] = allTimings[i].*field; + } + return v; + }; + + auto prepareTxSet = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::prepareTxSetMs); + auto prefetchSrc = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::prefetchSourceAccountsMs); + auto feesSeqNums = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::processFeesSeqNumsMs); + auto applyTxs = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::applyTransactionsMs); + auto applyTxSetup = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::applyTxSetupMs); + auto prefetchTxData = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::prefetchTxDataMs); + auto applyTxMidSetup = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::applyTxMidSetupMs); + auto loadSorobanConfig = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::loadSorobanConfigMs); + auto buildTxBundles = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::buildTxBundlesMs); + auto sorobanSetupGlobal = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::sorobanSetupGlobalMs); + auto sorobanParallel = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::sorobanParallelApplyMs); + auto sorobanCheckInvariants = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::sorobanCheckInvariantsMs); + auto sorobanCommitThreads = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings:: + sorobanCommitFromThreadsMs); + auto sorobanDestroyThreads = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings:: + sorobanDestroyThreadStatesMs); + auto sorobanCommitLtx = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::sorobanCommitToLtxMs); + auto sorobanDestroyGlobal = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings:: + sorobanDestroyGlobalStateMs); + auto parTotal = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::applyParallelPhaseTotalMs); + auto applySeqClassic = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::applySeqClassicMs); + auto postTxSetApply = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::postTxSetApplyMs); + auto applyTxTail = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::applyTxTailMs); + auto destroyApplyStages = extract( + &LedgerManagerImpl::LedgerClosePhaseTimings::destroyApplyStagesMs); + auto upgrades = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::applyUpgradesMs); + auto sealBucket = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::sealAndBucketMs); + auto sqlCommit = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::sqlCommitMs); + auto postCommit = + extract(&LedgerManagerImpl::LedgerClosePhaseTimings::postCommitMs); + + // Compute per-ledger gap inside parallel_total: + // parallel_total - sum(all sub-phases including destructors) + std::vector parGap(n); + for (size_t i = 0; i < n; ++i) + { + parGap[i] = parTotal[i] - buildTxBundles[i] - sorobanSetupGlobal[i] - + sorobanParallel[i] - sorobanCheckInvariants[i] - + sorobanCommitThreads[i] - sorobanDestroyThreads[i] - + sorobanCommitLtx[i] - sorobanDestroyGlobal[i]; + } + // Compute per-ledger gap inside apply_transactions: + // apply_transactions - sum(all sub-phases including destructors) + std::vector txGap(n); + for (size_t i = 0; i < n; ++i) + { + txGap[i] = applyTxs[i] - applyTxSetup[i] - prefetchTxData[i] - + applyTxMidSetup[i] - loadSorobanConfig[i] - parTotal[i] - + applySeqClassic[i] - postTxSetApply[i] - applyTxTail[i] - + destroyApplyStages[i]; + } + + struct PhaseRow + { + std::string name; + PhaseStats stats; + }; + + // Hierarchical layout: + // Level 0: top-level phases (no indent) + // Level 1: children of apply_transactions (2-space indent) + // Level 2: children of parallel_total (4-space indent) + std::vector rows = { + {"prepare_txset", computePhaseStats(prepareTxSet)}, + {"prefetch_src_accts", computePhaseStats(prefetchSrc)}, + {"process_fees_seqnums", computePhaseStats(feesSeqNums)}, + {"apply_transactions", computePhaseStats(applyTxs)}, + {"| setup", computePhaseStats(applyTxSetup)}, + {"| prefetch_tx_data", computePhaseStats(prefetchTxData)}, + {"| mid_setup", computePhaseStats(applyTxMidSetup)}, + {"| load_soroban_config", computePhaseStats(loadSorobanConfig)}, + {"| parallel_total", computePhaseStats(parTotal)}, + {"| build_tx_bundles", computePhaseStats(buildTxBundles)}, + {"| soroban_setup_glbl", computePhaseStats(sorobanSetupGlobal)}, + {"| soroban_parallel", computePhaseStats(sorobanParallel)}, + {"| soroban_invariants", computePhaseStats(sorobanCheckInvariants)}, + {"| commit_from_thrds", computePhaseStats(sorobanCommitThreads)}, + {"| ~thread_states", computePhaseStats(sorobanDestroyThreads)}, + {"| commit_to_ltx", computePhaseStats(sorobanCommitLtx)}, + {"| ~global_par_state", computePhaseStats(sorobanDestroyGlobal)}, + {"| *** par gap ***", computePhaseStats(parGap)}, + {"| apply_seq_classic", computePhaseStats(applySeqClassic)}, + {"| post_tx_set_apply", computePhaseStats(postTxSetApply)}, + {"| tail", computePhaseStats(applyTxTail)}, + {"| ~apply_stages", computePhaseStats(destroyApplyStages)}, + {"| *** tx gap ***", computePhaseStats(txGap)}, + {"apply_upgrades", computePhaseStats(upgrades)}, + {"seal_and_bucket", computePhaseStats(sealBucket)}, + {"sql_commit", computePhaseStats(sqlCommit)}, + {"post_commit", computePhaseStats(postCommit)}, + }; + + // Log the table header and rows. + CLOG_WARNING(Perf, + "Phase timing breakdown ({} ledgers, all values in ms):", n); + CLOG_WARNING( + Perf, "{:<24s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}", + "phase", "mean", "stddev", "median", "p25", "p75", "p95", "p99"); + CLOG_WARNING( + Perf, + "{:-<24s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s}", "", + "", "", "", "", "", "", ""); + for (auto const& r : rows) + { + CLOG_WARNING(Perf, + "{:<24s} {:>8.2f} {:>8.2f} {:>8.2f} {:>8.2f} {:>8.2f} " + "{:>8.2f} {:>8.2f}", + r.name, r.stats.mean, r.stats.stddev, r.stats.median, + r.stats.p25, r.stats.p75, r.stats.p95, r.stats.p99); + } +} + +void +logTxSetBuildTimingsTable(std::vector const& allTimings) +{ + if (allTimings.empty()) + { + return; + } + + size_t n = allTimings.size(); + auto extract = [&](auto field) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) + { + v[i] = allTimings[i].*field; + } + return v; + }; + + auto total = extract(&TxSetBuildPhaseTimings::totalMs); + auto trimClassic = extract(&TxSetBuildPhaseTimings::trimInvalidClassicMs); + auto surgeClassic = extract(&TxSetBuildPhaseTimings::surgePricingClassicMs); + auto trimSoroban = extract(&TxSetBuildPhaseTimings::trimInvalidSorobanMs); + auto surgeSoroban = extract(&TxSetBuildPhaseTimings::surgePricingSorobanMs); + auto parallelBuild = + extract(&TxSetBuildPhaseTimings::buildParallelSorobanPhaseMs); + auto buildApplicable = + extract(&TxSetBuildPhaseTimings::buildApplicableTxSetMs); + auto toWire = extract(&TxSetBuildPhaseTimings::toWireTxSetMs); + auto prepareForApply = + extract(&TxSetBuildPhaseTimings::prepareTxSetForApplyMs); + auto validateShape = + extract(&TxSetBuildPhaseTimings::validateRoundTripShapeMs); + auto validateTxSet = extract(&TxSetBuildPhaseTimings::validateTxSetMs); + + std::vector classicTotal(n); + std::vector sorobanTotal(n); + std::vector sorobanSurgeGap(n); + std::vector totalGap(n); + for (size_t i = 0; i < n; ++i) + { + classicTotal[i] = trimClassic[i] + surgeClassic[i]; + sorobanTotal[i] = trimSoroban[i] + surgeSoroban[i]; + sorobanSurgeGap[i] = surgeSoroban[i] - parallelBuild[i]; + totalGap[i] = total[i] - classicTotal[i] - sorobanTotal[i] - + buildApplicable[i] - toWire[i] - prepareForApply[i] - + validateShape[i] - validateTxSet[i]; + } + + struct PhaseRow + { + std::string name; + PhaseStats stats; + }; + + std::vector rows = { + {"total", computePhaseStats(total)}, + {"phase_classic", computePhaseStats(classicTotal)}, + {"| trim_invalid", computePhaseStats(trimClassic)}, + {"| surge_pricing", computePhaseStats(surgeClassic)}, + {"phase_soroban", computePhaseStats(sorobanTotal)}, + {"| trim_invalid", computePhaseStats(trimSoroban)}, + {"| surge_pricing", computePhaseStats(surgeSoroban)}, + {"| parallel_build", computePhaseStats(parallelBuild)}, + {"| *** soroban gap ***", computePhaseStats(sorobanSurgeGap)}, + {"build_applicable", computePhaseStats(buildApplicable)}, + {"to_wire", computePhaseStats(toWire)}, + {"prepare_for_apply", computePhaseStats(prepareForApply)}, + {"validate_shape", computePhaseStats(validateShape)}, + {"validate_txset", computePhaseStats(validateTxSet)}, + {"*** txset gap ***", computePhaseStats(totalGap)}, + }; + + CLOG_WARNING( + Perf, + "Tx-set build timing breakdown ({} ledgers, all values in ms):", n); + CLOG_WARNING( + Perf, "{:<28s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}", + "phase", "mean", "stddev", "median", "p25", "p75", "p95", "p99"); + CLOG_WARNING( + Perf, + "{:-<28s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s}", "", + "", "", "", "", "", "", ""); + for (auto const& r : rows) + { + CLOG_WARNING(Perf, + "{:<28s} {:>8.2f} {:>8.2f} {:>8.2f} {:>8.2f} {:>8.2f} " + "{:>8.2f} {:>8.2f}", + r.name, r.stats.mean, r.stats.stddev, r.stats.median, + r.stats.p25, r.stats.p75, r.stats.p95, r.stats.p99); + } +} + SorobanUpgradeConfig getUpgradeConfig(Config const& cfg, bool validate = true) { @@ -795,9 +1081,11 @@ ApplyLoad::setup() void ApplyLoad::closeLedger(std::vector const& txs, xdr::xvector const& upgrades, - bool recordSorobanUtilization) + bool recordSorobanUtilization, + TxSetBuildPhaseTimings* txSetBuildTimings) { - auto txSet = makeTxSetFromTransactions(txs, mApp, 0, 0); + auto txSet = makeTxSetFromTransactions(txs, mApp, 0, 0, false, {}, + txSetBuildTimings); if (recordSorobanUtilization) { @@ -1887,32 +2175,46 @@ ApplyLoad::benchmarkModelTx() std::vector closeTimes; closeTimes.reserve(config.APPLY_LOAD_NUM_LEDGERS); + // Per-phase timing vectors + using Timings = LedgerManagerImpl::LedgerClosePhaseTimings; + std::vector allPhaseTimings; + allPhaseTimings.reserve(config.APPLY_LOAD_NUM_LEDGERS); + std::vector allTxSetBuildTimings; + allTxSetBuildTimings.reserve(config.APPLY_LOAD_NUM_LEDGERS); + CLOG_WARNING(Perf, "Starting model transaction benchmark for {} ledgers with " "{} tx per ledger", config.APPLY_LOAD_NUM_LEDGERS, config.APPLY_LOAD_MAX_SOROBAN_TX_COUNT); + auto& lm = static_cast(mApp.getLedgerManager()); + for (size_t i = 0; i < config.APPLY_LOAD_NUM_LEDGERS; ++i) { double closeTimeMs = 0.0; + TxSetBuildPhaseTimings txSetBuildTimings; switch (mModelTx) { case ApplyLoadModelTx::SAC: closeTimeMs = benchmarkModelTxTpsSingleLedger( - ApplyLoadModelTx::SAC, calculateBenchmarkModelTxCount()); + ApplyLoadModelTx::SAC, calculateBenchmarkModelTxCount(), + &txSetBuildTimings); break; case ApplyLoadModelTx::CUSTOM_TOKEN: closeTimeMs = benchmarkModelTxTpsSingleLedger( ApplyLoadModelTx::CUSTOM_TOKEN, - calculateBenchmarkModelTxCount()); + calculateBenchmarkModelTxCount(), &txSetBuildTimings); break; case ApplyLoadModelTx::SOROSWAP: closeTimeMs = benchmarkModelTxTpsSingleLedger( - ApplyLoadModelTx::SOROSWAP, calculateBenchmarkModelTxCount()); + ApplyLoadModelTx::SOROSWAP, calculateBenchmarkModelTxCount(), + &txSetBuildTimings); break; } closeTimes.emplace_back(closeTimeMs); + allPhaseTimings.emplace_back(lm.getLastPhaseTimings()); + allTxSetBuildTimings.emplace_back(txSetBuildTimings); } releaseAssert(!closeTimes.empty()); @@ -1949,11 +2251,16 @@ ApplyLoad::benchmarkModelTx() interpolatePercentile(sortedCloseTimes, 99.0)); CLOG_WARNING(Perf, "close time stddev: {} ms", std::sqrt(varianceMsSq)); CLOG_WARNING(Perf, "================================================"); + + // Compute and output per-phase statistics table. + logPhaseTimingsTable(allPhaseTimings); + logTxSetBuildTimingsTable(allTxSetBuildTimings); } double -ApplyLoad::benchmarkModelTxTpsSingleLedger(ApplyLoadModelTx modelTx, - uint32_t txsPerLedger) +ApplyLoad::benchmarkModelTxTpsSingleLedger( + ApplyLoadModelTx modelTx, uint32_t txsPerLedger, + TxSetBuildPhaseTimings* txSetBuildTimings) { auto& totalTxApplyTimer = mApp.getConfig().APPLY_LOAD_TIME_WRITES @@ -1998,7 +2305,7 @@ ApplyLoad::benchmarkModelTxTpsSingleLedger(ApplyLoadModelTx modelTx, releaseAssert( mApp.getBucketManager().getHotArchiveBucketList().futuresAllResolved()); double timeBefore = totalTxApplyTimer.sum(); - closeLedger(txs); + closeLedger(txs, {}, false, txSetBuildTimings); double timeAfter = totalTxApplyTimer.sum(); double closeTime = timeAfter - timeBefore; diff --git a/src/simulation/ApplyLoad.h b/src/simulation/ApplyLoad.h index d16c200f44..76f1a120f9 100644 --- a/src/simulation/ApplyLoad.h +++ b/src/simulation/ApplyLoad.h @@ -10,6 +10,8 @@ namespace stellar { +struct TxSetBuildPhaseTimings; + class ApplyLoad { public: @@ -28,7 +30,8 @@ class ApplyLoad // the benchmark runs. void closeLedger(std::vector const& txs, xdr::xvector const& upgrades = {}, - bool recordSorobanUtilization = false); + bool recordSorobanUtilization = false, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr); // These metrics track what percentage of available resources were used when // creating the list of transactions in benchmark(). @@ -92,8 +95,9 @@ class ApplyLoad // Run a single ledger benchmark at the given TPS. Returns the close time // in milliseconds for that ledger. - double benchmarkModelTxTpsSingleLedger(ApplyLoadModelTx modelTx, - uint32_t txsPerLedger); + double benchmarkModelTxTpsSingleLedger( + ApplyLoadModelTx modelTx, uint32_t txsPerLedger, + TxSetBuildPhaseTimings* txSetBuildTimings = nullptr); // Run a single ledger benchmark for the model transaction mode. Returns // the close time in milliseconds for that ledger. diff --git a/src/test/TestUtils.cpp b/src/test/TestUtils.cpp index 52c122d833..08d25830ee 100644 --- a/src/test/TestUtils.cpp +++ b/src/test/TestUtils.cpp @@ -301,7 +301,7 @@ prepareSorobanNetworkConfigUpgrade( auto root = app.getRoot(); auto closeWithTx = [&](TransactionFrameBaseConstPtr tx) { - auto res = txtest::closeLedgerOn( + txtest::closeLedgerOn( app, app.getLedgerManager().getLastClosedLedgerNum() + 1, 2, 1, 2016, {tx}); root->loadSequenceNumber(); diff --git a/src/transactions/FeeBumpTransactionFrame.cpp b/src/transactions/FeeBumpTransactionFrame.cpp index ab80153ab6..647d1bd2bf 100644 --- a/src/transactions/FeeBumpTransactionFrame.cpp +++ b/src/transactions/FeeBumpTransactionFrame.cpp @@ -90,10 +90,10 @@ FeeBumpTransactionFrame::preParallelApply( { try { - LedgerTxn ltxTx(ltx); - removeOneTimeSignerKeyFromFeeSource(ltxTx); - meta.pushTxChangesBefore(ltxTx); - ltxTx.commit(); + ParallelPreApplyInfo info; + LedgerSnapshot ls(ltx); + preParallelApplyReadOnly(app, ls, meta, txResult, sorobanConfig, info); + preParallelApplyWrite(app, ltx, meta, info); } catch (std::exception& e) { @@ -103,19 +103,54 @@ FeeBumpTransactionFrame::preParallelApply( { printErrorAndAbort("Unknown exception in preParallelApply"); } +} + +void +FeeBumpTransactionFrame::preParallelApplyReadOnly( + AppConnector& app, LedgerSnapshot const& ls, TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, ParallelPreApplyInfo& info) const +{ + try + { + mInnerTx->preParallelApplyReadOnly(/*chargeFee=*/false, app, ls, meta, + txResult, sorobanConfig, + getContentsHash(), info); + } + catch (std::exception& e) + { + printErrorAndAbort("Exception during read-only preParallelApply: ", + e.what()); + } + catch (...) + { + printErrorAndAbort( + "Unknown exception during read-only preParallelApply"); + } +} +void +FeeBumpTransactionFrame::preParallelApplyWrite( + AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const +{ try { - mInnerTx->preParallelApply(/*chargeFee=*/false, app, ltx, meta, - txResult, sorobanConfig, getContentsHash()); + LedgerTxn ltxTx(ltx); + removeOneTimeSignerKeyFromFeeSource(ltxTx); + meta.pushTxChangesBefore(ltxTx); + ltxTx.commit(); + + mInnerTx->preParallelApplyWrite(app, ltx, meta, info); } catch (std::exception& e) { - printErrorAndAbort("Exception during preParallelApply: ", e.what()); + printErrorAndAbort("Exception during preParallelApply writes: ", + e.what()); } catch (...) { - printErrorAndAbort("Unknown exception during preParallelApply"); + printErrorAndAbort("Unknown exception during preParallelApply writes"); } } @@ -328,6 +363,46 @@ FeeBumpTransactionFrame::checkValid( return txResult; } +MutableTxResultPtr +FeeBumpTransactionFrame::checkValid( + AppConnector& app, LedgerSnapshot const& ls, SequenceNumber current, + uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const +{ + if (!xdr::check_xdr_depth(mEnvelope, 500) || !XDRProvidesValidFee()) + { + return FeeBumpMutableTransactionResult::createTxError(txMALFORMED); + } + + int64_t minBaseFee = ls.getLedgerHeader().current().baseFee; + auto feeCharged = getFee(ls.getLedgerHeader().current(), minBaseFee, false); + auto txResult = FeeBumpMutableTransactionResult::createSuccess( + *mInnerTx, feeCharged, 0); + + SignatureChecker signatureChecker{ + ls.getLedgerHeader().current().ledgerVersion, getContentsHash(), + mEnvelope.feeBump().signatures}; + if (commonValid(signatureChecker, ls, false, *txResult) != + ValidationType::kFullyValid) + { + return txResult; + } + + if (!signatureChecker.checkAllSignaturesUsed()) + { + txResult->setError(txBAD_AUTH_EXTRA); + return txResult; + } + + mInnerTx->checkValidWithOptionallyChargedFee( + app, ls, current, false, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, getContentsHash(), *txResult, + diagnosticEvents, sorobanConfig); + + return txResult; +} + bool FeeBumpTransactionFrame::checkSorobanResources( SorobanNetworkConfig const& cfg, uint32_t ledgerVersion, diff --git a/src/transactions/FeeBumpTransactionFrame.h b/src/transactions/FeeBumpTransactionFrame.h index 77596327ff..331c4dda33 100644 --- a/src/transactions/FeeBumpTransactionFrame.h +++ b/src/transactions/FeeBumpTransactionFrame.h @@ -87,6 +87,16 @@ class FeeBumpTransactionFrame : public TransactionFrameBase MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApplyReadOnly(AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + ParallelPreApplyInfo& info) const override; + + void preParallelApplyWrite(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const override; + std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, @@ -114,6 +124,12 @@ class FeeBumpTransactionFrame : public TransactionFrameBase SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, DiagnosticEventManager& diagnosticEvents) const override; + MutableTxResultPtr + checkValid(AppConnector& app, LedgerSnapshot const& ls, + SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const override; bool checkSorobanResources( SorobanNetworkConfig const& cfg, uint32_t ledgerVersion, DiagnosticEventManager& diagnosticEvents) const override; diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 7aa68bdc4d..1d6710f511 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -11,6 +11,7 @@ #include "util/ProtocolVersion.h" #include "xdr/Stellar-ledger-entries.h" #include +#include #include #include #include "xdr/Stellar-contract.h" @@ -18,6 +19,7 @@ #include "ledger/LedgerTxnImpl.h" #include "rust/CppShims.h" +#include "util/BitSet.h" #include "xdr/Stellar-transaction.h" #include #include @@ -39,9 +41,10 @@ namespace stellar namespace { CxxLedgerInfo -getLedgerInfo(SorobanNetworkConfig const& sorobanConfig, uint32_t ledgerVersion, - uint32_t ledgerSeq, uint32_t baseReserve, TimePoint closeTime, - Hash const& networkID) +buildLedgerInfo(SorobanNetworkConfig const& sorobanConfig, + uint32_t ledgerVersion, uint32_t ledgerSeq, + uint32_t baseReserve, TimePoint closeTime, + Hash const& networkID) { CxxLedgerInfo info{}; info.base_reserve = baseReserve; @@ -69,6 +72,27 @@ getLedgerInfo(SorobanNetworkConfig const& sorobanConfig, uint32_t ledgerVersion, return info; } +CxxLedgerInfo const& +getCachedLedgerInfo(SorobanNetworkConfig const& sorobanConfig, + uint32_t ledgerVersion, uint32_t ledgerSeq, + uint32_t baseReserve, TimePoint closeTime, + Hash const& networkID) +{ + thread_local std::optional cachedLedgerSeq; + thread_local std::optional cachedLedgerInfo; + + if (!cachedLedgerSeq || *cachedLedgerSeq != ledgerSeq) + { + cachedLedgerSeq = ledgerSeq; + cachedLedgerInfo = + buildLedgerInfo(sorobanConfig, ledgerVersion, ledgerSeq, + baseReserve, closeTime, networkID); + } + + releaseAssertOrThrow(cachedLedgerInfo); + return cachedLedgerInfo.value(); +} + DiagnosticEvent metricsEvent(bool success, std::string&& topic, uint64_t value) { @@ -270,6 +294,7 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper rust::Vec mLedgerEntryCxxBufs; rust::Vec mTtlEntryCxxBufs; rust::Vec mAutoRestoredRwEntryIndices; + BitSet mRwKeyExisted; HostFunctionMetrics mMetrics; // Used for hot archive access only ApplyLedgerStateSnapshot mStateSnapshot; @@ -297,6 +322,7 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper , mResources(mOpFrame.mParentTx.sorobanResources()) , mSorobanConfig(sorobanConfig) , mAppConfig(app.getConfig()) + , mRwKeyExisted(mResources.footprint.readWrite.size()) , mMetrics(app.getSorobanMetrics(), app.getConfig().DISABLE_SOROBAN_METRICS_FOR_TESTING) , mStateSnapshot(std::move(stateSnapshot)) @@ -313,7 +339,7 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper mTtlEntryCxxBufs.reserve(footprintLength); } - virtual CxxLedgerInfo getLedgerInfo() = 0; + virtual CxxLedgerInfo const& getLedgerInfo() = 0; // Helper called on all archived keys in the footprint. Returns false if // the operation should fail and populates result code and diagnostic @@ -450,6 +476,11 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper auto entryOpt = getLedgerEntryOpt(lk); if (entryOpt) { + if (!isReadOnly) + { + mRwKeyExisted.set(i); + } + auto leBuf = toCxxBuf(*entryOpt); entrySize = static_cast(leBuf.data->size()); @@ -610,14 +641,23 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper recordStorageChanges(InvokeHostFunctionOutput const& out) { ZoneScoped; - // Create or update every entry returned. - UnorderedSet createdAndModifiedKeys; - UnorderedSet createdKeys; + // Track which RW footprint keys appear in the host output without + // hashing LedgerKeys. Footprints are small, so a linear scan over a + // BitSet-backed coverage map is cheaper than maintaining hash sets. + auto const& rwKeys = mResources.footprint.readWrite; + BitSet rwKeyCovered(rwKeys.size()); + size_t numCreatedSorobanEntries = 0; + size_t numCreatedTTLEntries = 0; + bool const allowClassicCreations = protocolVersionStartsFrom( + getLedgerVersion(), ProtocolVersion::V_26); + for (auto const& buf : out.modified_ledger_entries) { LedgerEntry le; xdr::xdr_from_opaque(buf.data, le); auto lk = LedgerEntryKey(le); + size_t matchedRwKey = rwKeys.size(); + size_t relatedRwKey = rwKeys.size(); if (!validateContractLedgerEntry( lk, buf.data.size(), mSorobanConfig, mAppConfig, mOpFrame.mParentTx, mDiagnosticEvents)) @@ -627,15 +667,38 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper return false; } - createdAndModifiedKeys.insert(lk); - - uint32_t keySize = static_cast(xdr::xdr_size(lk)); uint32_t entrySize = static_cast(buf.data.size()); + for (size_t j = 0; j < rwKeys.size(); ++j) + { + bool directMatch = rwKeys[j] == lk; + if (directMatch) + { + relatedRwKey = j; + if (!rwKeyCovered.get(j)) + { + rwKeyCovered.set(j); + matchedRwKey = j; + } + } + else if (lk.type() == TTL && isSorobanEntry(rwKeys[j]) && + getTTLKey(rwKeys[j]) == lk) + { + relatedRwKey = j; + } + + if (matchedRwKey != rwKeys.size() && + relatedRwKey != rwKeys.size()) + { + break; + } + } + // ttlEntry write fees come out of refundableFee, already // accounted for by the host if (lk.type() != TTL) { + uint32_t keySize = static_cast(xdr::xdr_size(lk)); mMetrics.noteWriteEntry(isContractCodeEntry(lk), keySize, entrySize); if (mResources.writeBytes < mMetrics.mLedgerWriteByte) @@ -652,44 +715,44 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper } } - if (upsertLedgerEntry(lk, le)) + bool created = relatedRwKey != rwKeys.size() && + !mRwKeyExisted.get(relatedRwKey); + upsertLedgerEntry(lk, le); + if (created) { - createdKeys.insert(lk); + if (isSorobanEntry(lk)) + { + ++numCreatedSorobanEntries; + } + else if (lk.type() == TTL) + { + ++numCreatedTTLEntries; + } + else if (allowClassicCreations) + { + releaseAssertOrThrow(lk.type() == ACCOUNT || + lk.type() == TRUSTLINE); + } + else + { + releaseAssertOrThrow(false); + } } } - // Check that each newly created ContractCode or ContractData entry also - // creates a ttlEntry. Starting from protocol 26 (CAP-73), the Stellar - // Asset Contract can also create classic entries (ACCOUNT, TRUSTLINE). - for (auto const& key : createdKeys) - { - if (isSorobanEntry(key)) - { - auto ttlKey = getTTLKey(key); - releaseAssertOrThrow(createdKeys.find(ttlKey) != - createdKeys.end()); - } - else if (protocolVersionStartsFrom(getLedgerVersion(), - ProtocolVersion::V_26)) - { - releaseAssertOrThrow(key.type() == TTL || - key.type() == ACCOUNT || - key.type() == TRUSTLINE); - } - else - { - releaseAssertOrThrow(key.type() == TTL); - } - } + // Verify that each newly created Soroban entry has a corresponding + // newly created TTL entry (1:1 pairing guaranteed by the host). + releaseAssertOrThrow(numCreatedSorobanEntries == numCreatedTTLEntries); // Erase every entry not returned. // NB: The entries that haven't been touched are passed through // from host, so this should never result in removing an entry // that hasn't been removed by host explicitly. - for (auto const& lk : mResources.footprint.readWrite) + for (size_t j = 0; j < rwKeys.size(); ++j) { - if (createdAndModifiedKeys.find(lk) == createdAndModifiedKeys.end()) + if (!rwKeyCovered.get(j)) { + auto const& lk = rwKeys[j]; if (eraseLedgerEntryIfExists(lk)) { releaseAssertOrThrow(isSorobanEntry(lk)); @@ -818,7 +881,43 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper { xdr::xdr_from_opaque(out.result_value.data, success.returnValue); mOpFrame.innerResult(mRes).code(INVOKE_HOST_FUNCTION_SUCCESS); - mOpFrame.innerResult(mRes).success() = xdrSha256(success); + + // Streaming SHA256 calculation of xdrSha256(success) + // This avoids round-trip serialization of the potentially large + // `InvokeHostFunctionSuccessPreImage` struct, which is significant for + // large return values or many contract events. + // + // The structure being hashed is `InvokeHostFunctionSuccessPreImage`, + // defined as: struct InvokeHostFunctionSuccessPreImage { + // SCVal returnValue; + // ContractEvent events<>; + // }; + // + // XDR encoding of this struct is: + // 1. returnValue (SCVal) + // 2. events (array of ContractEvent) + // - length (uint32) + // - [ContractEvent, ContractEvent, ...] + + SHA256 hasher; + + // 1. Add returnValue (SCVal) + // out.result_value.data is already the XDR encoded bytes of returnValue + hasher.add(out.result_value.data); + + // 2. Add events length (uint32) + uint32_t eventsSize = static_cast(out.contract_events.size()); + uint32_t eventsSizeNet = htonl(eventsSize); + hasher.add(ByteSlice(&eventsSizeNet, sizeof(eventsSizeNet))); + + // 3. Add each event + for (auto const& buf : out.contract_events) + { + // buf.data is already the XDR encoded bytes of the ContractEvent + hasher.add(buf.data); + } + + mOpFrame.innerResult(mRes).success() = hasher.finish(); // success.events is moved in setEvents, so don't use it after this // call. @@ -971,14 +1070,14 @@ class InvokeHostFunctionPreV23ApplyHelper return false; } - CxxLedgerInfo + CxxLedgerInfo const& getLedgerInfo() override { auto hdr = mLtx.loadHeader(); auto const& lh = hdr.current(); - return stellar::getLedgerInfo( - mSorobanConfig, lh.ledgerVersion, lh.ledgerSeq, lh.baseReserve, - lh.scpValue.closeTime, mApp.getNetworkID()); + return getCachedLedgerInfo(mSorobanConfig, lh.ledgerVersion, + lh.ledgerSeq, lh.baseReserve, + lh.scpValue.closeTime, mApp.getNetworkID()); } public: @@ -1158,10 +1257,10 @@ class InvokeHostFunctionParallelApplyHelper return mAutorestoredEntries.at(index); } - CxxLedgerInfo + CxxLedgerInfo const& getLedgerInfo() override { - return stellar::getLedgerInfo( + return getCachedLedgerInfo( mSorobanConfig, mLedgerInfo.getLedgerVersion(), mLedgerInfo.getLedgerSeq(), mLedgerInfo.getBaseReserve(), mLedgerInfo.getCloseTime(), mLedgerInfo.getNetworkID()); diff --git a/src/transactions/ParallelApplyStage.h b/src/transactions/ParallelApplyStage.h index 4760384637..eaca2ce213 100644 --- a/src/transactions/ParallelApplyStage.h +++ b/src/transactions/ParallelApplyStage.h @@ -36,6 +36,18 @@ class TxEffects return mDelta; } + ParallelPreApplyInfo& + getParallelPreApplyInfo() + { + return mParallelPreApplyInfo; + } + + ParallelPreApplyInfo const& + getParallelPreApplyInfo() const + { + return mParallelPreApplyInfo; + } + void setDeltaEntry(LedgerKey const& key, LedgerTxnDelta::EntryDelta const& delta) { @@ -53,6 +65,7 @@ class TxEffects private: TransactionMetaBuilder mMeta; LedgerTxnDelta mDelta; + ParallelPreApplyInfo mParallelPreApplyInfo; }; // TxBundle contains a transaction, its associated result payload, and its diff --git a/src/transactions/ParallelApplyUtils.cpp b/src/transactions/ParallelApplyUtils.cpp index 12f3772484..d2937b98f7 100644 --- a/src/transactions/ParallelApplyUtils.cpp +++ b/src/transactions/ParallelApplyUtils.cpp @@ -8,13 +8,18 @@ #include "ledger/LedgerTxn.h" #include "ledger/NetworkConfig.h" #include "main/AppConnector.h" +#include "transactions/OperationFrame.h" #include "transactions/ParallelApplyStage.h" #include "transactions/TransactionFrameBase.h" +#include "transactions/TransactionUtils.h" #include "util/GlobalChecks.h" +#include "util/ProtocolVersion.h" #include "xdr/Stellar-ledger-entries.h" #include "xdrpp/printer.h" +#include #include #include +#include #include namespace @@ -96,11 +101,20 @@ using namespace stellar; // total order, B could save this fee, but we would lose the ability to run A // and B in parallel in the future. CAP 0063 explicitly chose this tradeoff. -std::unordered_set +ParallelApplyLedgerKeySet getReadWriteKeysForStage(ApplyStage const& stage) { ZoneScoped; - std::unordered_set res; + ParallelApplyLedgerKeySet res; + + // Pre-reserve to avoid rehashing. Each RW key may also have a TTL key. + size_t estimatedKeys = 0; + for (auto const& txBundle : stage) + { + estimatedKeys += + txBundle.getTx()->sorobanResources().footprint.readWrite.size() * 2; + } + res.reserve(estimatedKeys); for (auto const& txBundle : stage) { @@ -117,6 +131,82 @@ getReadWriteKeysForStage(ApplyStage const& stage) return res; } +void +readOnlyPreParallelApplyRange(AppConnector& app, + ApplyLedgerStateSnapshot const& snapshot, + std::vector const& txBundles, + size_t begin, size_t end, + SorobanNetworkConfig const& sorobanConfig) +{ + LedgerSnapshot ls(snapshot); + for (size_t i = begin; i < end; ++i) + { + auto const& txBundle = *txBundles.at(i); + txBundle.getTx()->preParallelApplyReadOnly( + app, ls, txBundle.getEffects().getMeta(), txBundle.getResPayload(), + sorobanConfig, txBundle.getEffects().getParallelPreApplyInfo()); + } +} + +bool +isModifiedClassicKey(LedgerSnapshot const& current, + LedgerSnapshot const& previous, LedgerKey const& key) +{ + if (isSorobanEntry(key)) + { + return false; + } + + auto currentEntry = current.load(key); + auto previousEntry = previous.load(key); + if (static_cast(currentEntry) != static_cast(previousEntry)) + { + return true; + } + + return currentEntry && currentEntry.current() != previousEntry.current(); +} + +bool +requiresSequentialPreParallelApply(LedgerSnapshot const& current, + LedgerSnapshot const& previous, + TransactionFrameBase const& tx) +{ + if (isModifiedClassicKey(current, previous, accountKey(tx.getSourceID())) || + isModifiedClassicKey(current, previous, + accountKey(tx.getFeeSourceID()))) + { + return true; + } + + for (auto const& op : tx.getOperationFrames()) + { + if (isModifiedClassicKey(current, previous, + accountKey(op->getSourceID()))) + { + return true; + } + } + + auto const& footprint = tx.sorobanResources().footprint; + for (auto const& key : footprint.readOnly) + { + if (isModifiedClassicKey(current, previous, key)) + { + return true; + } + } + for (auto const& key : footprint.readWrite) + { + if (isModifiedClassicKey(current, previous, key)) + { + return true; + } + } + + return false; +} + inline uint32_t& ttl(LedgerEntry& le) { @@ -145,10 +235,10 @@ ttl(std::optional const& le) // (code-or-data) keys named in the footprint of the `txBundle`. Note // that since RO and RW footprints are disjoint, we only have to look // at the RO set. -UnorderedSet +ParallelApplyLedgerKeySet buildRoTTLSet(TxBundle const& txBundle) { - UnorderedSet isReadOnlyTTLSet; + ParallelApplyLedgerKeySet isReadOnlyTTLSet; for (auto const& ro : txBundle.getTx()->sorobanResources().footprint.readOnly) { @@ -164,10 +254,11 @@ buildRoTTLSet(TxBundle const& txBundle) // Accumulate into the buffer of `roTTLBumps` the max of any existing entry and // the provided `updatedLE`, which must be a non-nullopt TTL LE. void -updateMaxOfRoTTLBump(UnorderedMap& roTTLBumps, +updateMaxOfRoTTLBump(ParallelApplyLedgerKeyMap& roTTLBumps, LedgerKey const& lk, LedgerEntry const& updatedLe) { - auto [it, emplaced] = roTTLBumps.emplace(lk, ttl(updatedLe)); + ParallelApplyLedgerKey parallelKey(lk); + auto [it, emplaced] = roTTLBumps.emplace(parallelKey, ttl(updatedLe)); if (!emplaced) { it->second = std::max(it->second, ttl(updatedLe)); @@ -207,7 +298,7 @@ PreV23LedgerAccessHelper::getLedgerSeq() return mLtx.loadHeader().current().ledgerSeq; } -bool +void PreV23LedgerAccessHelper::upsertLedgerEntry(LedgerKey const& key, LedgerEntry const& entry) { @@ -215,12 +306,10 @@ PreV23LedgerAccessHelper::upsertLedgerEntry(LedgerKey const& key, if (ltxe) { ltxe.current() = entry; - return false; } else { mLtx.create(entry); - return true; } } @@ -266,11 +355,11 @@ ParallelLedgerAccessHelper::getLedgerVersion() return mLedgerInfo.getLedgerVersion(); } -bool +void ParallelLedgerAccessHelper::upsertLedgerEntry(LedgerKey const& key, LedgerEntry const& entry) { - return mTxState.upsertEntry(key, entry, mLedgerInfo.getLedgerSeq()); + mTxState.upsertEntry(key, entry, mLedgerInfo.getLedgerSeq()); } bool @@ -309,6 +398,24 @@ GlobalParallelApplyLedgerState::GlobalParallelApplyLedgerState( releaseAssertOrThrow(ltx.getHeader().ledgerSeq == mLCLSnapshot.getLedgerSeq() + 1); + // Pre-reserve global entry map to avoid rehashing as entries accumulate + // from classic fee processing, Soroban RO pre-loading, and thread commits. + // Each footprint key may have an associated TTL key, plus one classic + // source account entry per TX. + { + size_t estimatedEntries = 0; + for (auto const& stage : stages) + { + for (auto const& txBundle : stage) + { + auto const& fp = txBundle.getTx()->sorobanResources().footprint; + estimatedEntries += + fp.readWrite.size() * 2 + fp.readOnly.size() * 2 + 1; + } + } + mGlobalEntryMap.reserve(estimatedEntries); + } + // From now on, we will be using globalState, liveSnapshots, and the // hotArchive to collect all entries. Before we continue though, we need to // load into the globalEntryMap any classic entries that have been modified @@ -330,6 +437,36 @@ GlobalParallelApplyLedgerState:: releaseAssert(threadIsMain() || app.threadIsType(Application::ThreadType::APPLY)); + if (protocolVersionStartsFrom(ltx.loadHeader().current().ledgerVersion, + ProtocolVersion::V_26)) + { + std::vector txBundles; + LedgerSnapshot current(ltx); + LedgerSnapshot previous(mLCLSnapshot); + for (auto const& stage : stages) + { + for (auto const& txBundle : stage) + { + if (requiresSequentialPreParallelApply(current, previous, + *txBundle.getTx())) + { + txBundle.getTx()->preParallelApply( + app, ltx, txBundle.getEffects().getMeta(), + txBundle.getResPayload(), mSorobanConfig); + } + else + { + txBundles.emplace_back(&txBundle); + } + } + } + + readOnlyPreParallelApply(app, txBundles); + commitBufferedPreParallelApplyWrites(app, ltx, txBundles); + collectModifiedClassicEntries(ltx, stages); + return; + } + auto fetchInMemoryClassicEntries = [&](xdr::xvector const& keys) { for (auto const& lk : keys) @@ -386,12 +523,207 @@ GlobalParallelApplyLedgerState:: } void -GlobalParallelApplyLedgerState::commitChangesToLedgerTxn( - AbstractLedgerTxn& ltx) const +GlobalParallelApplyLedgerState::readOnlyPreParallelApply( + AppConnector& app, std::vector const& txBundles) +{ + ZoneScoped; + + if (txBundles.empty()) + { + return; + } + + auto workerCount = std::min( + static_cast(app.getConfig().LEDGER_CLOSE_WORKER_THREADS), + txBundles.size()); + + if (workerCount == 1) + { + readOnlyPreParallelApplyRange(app, mLCLSnapshot, txBundles, 0, + txBundles.size(), mSorobanConfig); + return; + } + + std::vector> futures; + futures.reserve(workerCount); + + size_t begin = 0; + auto const baseChunkSize = txBundles.size() / workerCount; + auto const remainder = txBundles.size() % workerCount; + for (size_t workerIndex = 0; workerIndex < workerCount; ++workerIndex) + { + auto const chunkSize = + baseChunkSize + (workerIndex < remainder ? 1u : 0u); + auto const end = begin + chunkSize; + futures.emplace_back(std::async( + std::launch::async, readOnlyPreParallelApplyRange, std::ref(app), + std::cref(mLCLSnapshot), std::cref(txBundles), begin, end, + std::cref(mSorobanConfig))); + begin = end; + } + + for (auto& future : futures) + { + releaseAssert(future.valid()); + try + { + future.get(); + } + catch (std::exception const& e) + { + printErrorAndAbort("Exception during read-only preParallelApply: ", + e.what()); + } + catch (...) + { + printErrorAndAbort( + "Unknown exception during read-only preParallelApply"); + } + } +} + +void +GlobalParallelApplyLedgerState::commitBufferedPreParallelApplyWrites( + AppConnector& app, AbstractLedgerTxn& ltx, + std::vector const& txBundles) +{ + ZoneScoped; + + for (auto const* txBundle : txBundles) + { + txBundle->getTx()->preParallelApplyWrite( + app, ltx, txBundle->getEffects().getMeta(), + txBundle->getEffects().getParallelPreApplyInfo()); + } +} + +void +GlobalParallelApplyLedgerState::collectModifiedClassicEntries( + AbstractLedgerTxn& ltx, std::vector const& stages) +{ + ZoneScoped; + + std::unordered_set classicKeys; + for (auto const& stage : stages) + { + for (auto const& txBundle : stage) + { + auto const& footprint = + txBundle.getTx()->sorobanResources().footprint; + for (auto const& key : footprint.readWrite) + { + if (!isSorobanEntry(key)) + { + classicKeys.emplace(key); + } + } + for (auto const& key : footprint.readOnly) + { + if (!isSorobanEntry(key)) + { + classicKeys.emplace(key); + } + } + } + } + + for (auto const& lk : classicKeys) + { + auto entryPair = ltx.getNewestVersionBelowRoot(lk); + if (!entryPair.first) + { + continue; + } + + GlobalParApplyLedgerEntryOpt entry = scopeAdoptEntryOpt( + entryPair.second + ? std::make_optional(entryPair.second->ledgerEntry()) + : std::nullopt); + + mGlobalEntryMap.emplace(lk, GlobalParallelApplyEntry{entry, false}); + } + + // Pre-load Soroban read-only entries (and their TTLs) from + // InMemorySorobanState into the global entry map. Without this, + // every thread-level getLiveEntryOpt for a read-only Soroban key + // falls through to InMemorySorobanState::get() (involving hash + // computation and LedgerEntry copy). For workloads like SAC + // transfers where all TXs share the same read-only entries + // (contract instance), this saves thousands of redundant lookups + // per thread. + { + ZoneNamedN(fetchSorobanRoZone, + "fetchSorobanReadOnlyEntries from footprints", true); + for (auto const& stage : stages) + { + for (auto const& txBundle : stage) + { + for (auto const& lk : + txBundle.getTx()->sorobanResources().footprint.readOnly) + { + if (!isSorobanEntry(lk)) + { + continue; + } + if (mGlobalEntryMap.find(lk) != mGlobalEntryMap.end()) + { + continue; + } + + std::shared_ptr res; + if (InMemorySorobanState::isInMemoryType(lk)) + { + res = mInMemorySorobanState.get(lk); + } + else + { + res = mLCLSnapshot.loadLiveEntry(lk); + } + + if (res) + { + GlobalParApplyLedgerEntryOpt entry = + scopeAdoptEntryOpt(std::make_optional(*res)); + mGlobalEntryMap.emplace( + lk, GlobalParallelApplyEntry{entry, false}); + + // Also pre-load the TTL entry + auto ttlKey = getTTLKey(lk); + if (mGlobalEntryMap.find(ttlKey) == + mGlobalEntryMap.end()) + { + std::shared_ptr ttlRes; + if (InMemorySorobanState::isInMemoryType(ttlKey)) + { + ttlRes = mInMemorySorobanState.get(ttlKey); + } + else + { + ttlRes = mLCLSnapshot.loadLiveEntry(ttlKey); + } + if (ttlRes) + { + GlobalParApplyLedgerEntryOpt ttlEntry = + scopeAdoptEntryOpt( + std::make_optional(*ttlRes)); + mGlobalEntryMap.emplace( + ttlKey, + GlobalParallelApplyEntry{ttlEntry, false}); + } + } + } + } + } + } + } +} + +void +GlobalParallelApplyLedgerState::commitChangesToLedgerTxn(AbstractLedgerTxn& ltx) { ZoneScoped; LedgerTxn ltxInner(ltx); - for (auto const& [key, entry] : mGlobalEntryMap) + for (auto& [key, entry] : mGlobalEntryMap) { // Only update if dirty bit is set if (!entry.mIsDirty) @@ -399,26 +731,36 @@ GlobalParallelApplyLedgerState::commitChangesToLedgerTxn( continue; } - std::optional const& updatedLe = - entry.mLedgerEntry.readInScope(*this); - if (updatedLe) + // Move the LedgerEntry out of the scoped wrapper. This is safe + // because commitChangesToLedgerTxn is the final operation on the + // global state — it is destroyed immediately after this call. + auto movedLe = entry.mLedgerEntry.moveFromScope(*this); + if (movedLe) { - auto ltxe = ltxInner.load(key); - if (ltxe) + // Use the mIsNew flag tracked during the parallel apply phase to + // decide between createWithoutLoading (INIT) and + // updateWithoutLoading (LIVE). This avoids the expensive per-entry + // existence check (mInMemorySorobanState.get() does SHA256 per + // CONTRACT_DATA key, and getNewestVersionBelowRoot does a hash map + // lookup for classic entries). + InternalLedgerEntry ile(std::move(*movedLe)); + if (entry.mIsNew) { - ltxe.current() = *updatedLe; + ltxInner.createWithoutLoading(std::move(ile)); } else { - ltxInner.create(*updatedLe); + ltxInner.updateWithoutLoading(std::move(ile)); } } else { - auto ltxe = ltxInner.load(key); + // Delete case: use load() + erase() to maintain EXACT consistency. + // Deletes are rare in SAC transfers, so the cost is negligible. + auto ltxe = ltxInner.load(key.ledgerKey()); if (ltxe) { - ltxInner.erase(key); + ltxInner.erase(key.ledgerKey()); } } } @@ -478,9 +820,9 @@ GlobalParallelApplyLedgerState::getRestoredEntries() const bool GlobalParallelApplyLedgerState::maybeMergeRoTTLBumps( - LedgerKey const& key, GlobalParallelApplyEntry const& newEntry, + ParallelApplyLedgerKey const& key, GlobalParallelApplyEntry const& newEntry, GlobalParallelApplyEntry& oldEntry, - std::unordered_set const& readWriteSet) + ParallelApplyLedgerKeySet const& readWriteSet) { // Read Only bumps will always be updating a pre-existing value. TTL // creation (!oldEntry) or deletion (!newEntry) are write conflicts that @@ -490,7 +832,7 @@ GlobalParallelApplyLedgerState::maybeMergeRoTTLBumps( auto merged = false; oldEntry.mLedgerEntry.modifyInScope( *this, [&](std::optional& oldLe) { - if (newLe && oldLe && key.type() == TTL) + if (newLe && oldLe && key.ledgerKey().type() == TTL) { releaseAssertOrThrow(newLe.value().data.type() == TTL); releaseAssertOrThrow(oldLe.value().data.type() == TTL); @@ -499,6 +841,11 @@ GlobalParallelApplyLedgerState::maybeMergeRoTTLBumps( uint32_t const& newTTL = ttl(newLe); uint32_t& oldTTL = ttl(oldLe); oldTTL = std::max(oldTTL, newTTL); + // Propagate lastModifiedLedgerSeq from the thread's + // entry. This is necessary when the old entry was + // pre-loaded with a stale lastModifiedLedgerSeq. + oldLe.value().lastModifiedLedgerSeq = + newLe.value().lastModifiedLedgerSeq; merged = true; } } @@ -508,36 +855,51 @@ GlobalParallelApplyLedgerState::maybeMergeRoTTLBumps( void GlobalParallelApplyLedgerState::commitChangeFromThread( - ThreadParallelApplyLedgerState const& thread, LedgerKey const& key, - ThreadParallelApplyEntry const& parEntry, - std::unordered_set const& readWriteSet) + ThreadParallelApplyLedgerState const& thread, + ParallelApplyLedgerKey const& key, ThreadParallelApplyEntry&& parEntry, + ParallelApplyLedgerKeySet const& readWriteSet) { if (!parEntry.mIsDirty) { return; } - auto rescopedParEntry = parEntry.rescope(thread, *this); - auto [it, inserted] = mGlobalEntryMap.emplace(key, rescopedParEntry); - if (!inserted) + auto rescopedParEntry = std::move(parEntry).rescope(thread, *this); + auto it = mGlobalEntryMap.find(key); + if (it == mGlobalEntryMap.end()) + { + mGlobalEntryMap.emplace(key, std::move(rescopedParEntry)); + } + else { if (!maybeMergeRoTTLBumps(key, rescopedParEntry, it->second, readWriteSet)) { - it->second = rescopedParEntry; + // Preserve mIsNew from the first stage that touched this entry. + bool oldIsNew = it->second.mIsNew; + it->second = std::move(rescopedParEntry); + it->second.mIsNew = oldIsNew; + } + else + { + // The merge modified the entry value in-place. Mark it dirty + // so commitChangesToLedgerTxn writes it. This is necessary + // when the entry was pre-loaded (with mIsDirty=false) by the + // Soroban RO entry pre-loading in the constructor. + it->second.mIsDirty = true; } } } void GlobalParallelApplyLedgerState::commitChangesFromThread( - AppConnector& app, ThreadParallelApplyLedgerState const& thread, - std::unordered_set const& readWriteSet) + AppConnector& app, ThreadParallelApplyLedgerState& thread, + ParallelApplyLedgerKeySet const& readWriteSet) { ZoneScoped; thread.scopeDeactivate(); - for (auto const& [key, entry] : thread.getEntryMap()) + for (auto& [key, entry] : thread.getEntryMap()) { - commitChangeFromThread(thread, key, entry, readWriteSet); + commitChangeFromThread(thread, key, std::move(entry), readWriteSet); } mGlobalRestoredEntries.addRestoresFrom(thread.getRestoredEntries()); } @@ -567,6 +929,19 @@ ThreadParallelApplyLedgerState::collectClusterFootprintEntriesFromGlobal( releaseAssert(threadIsMain() || app.threadIsType(Application::ThreadType::APPLY)); + // Pre-reserve thread entry map to avoid rehashing during per-TX + // execution. Each footprint key may have an associated TTL key. + { + size_t estimatedEntries = 0; + for (auto const& txBundle : cluster) + { + auto const& fp = txBundle.getTx()->sorobanResources().footprint; + estimatedEntries += + fp.readWrite.size() * 2 + fp.readOnly.size() * 2; + } + mThreadEntryMap.reserve(estimatedEntries); + } + // As part of the initialization of this thread state, we need to // collect all the keys that are in the global state map. For any keys // we need not in the global state, we will fetch them from the live @@ -575,17 +950,20 @@ ThreadParallelApplyLedgerState::collectClusterFootprintEntriesFromGlobal( global.getGlobalEntryMap(); auto fetchFromGlobal = [&](LedgerKey const& key) { - if (mThreadEntryMap.find(key) != mThreadEntryMap.end()) + ParallelApplyLedgerKey parallelKey(key); + if (mThreadEntryMap.find(parallelKey) != mThreadEntryMap.end()) { return; } - auto entryIt = globalEntryMap.find(key); + auto entryIt = globalEntryMap.find(parallelKey); if (entryIt != globalEntryMap.end()) { - mThreadEntryMap.emplace( - key, ThreadParallelApplyEntry::clean(scopeAdoptEntryOptFrom( - entryIt->second.mLedgerEntry, global))); + auto threadEntry = ThreadParallelApplyEntry::clean( + scopeAdoptEntryOptFrom(entryIt->second.mLedgerEntry, global)); + // Propagate mIsNew from global so subsequent upserts preserve it. + threadEntry.mIsNew = entryIt->second.mIsNew; + mThreadEntryMap.emplace(std::move(parallelKey), threadEntry); } }; @@ -636,8 +1014,9 @@ ThreadParallelApplyLedgerState::flushRoTTLBumpsInTxWriteFootprint( continue; } - auto const& ttlKey = getTTLKey(lk); - auto b = mRoTTLBumps.find(ttlKey); + auto ttlKey = getTTLKey(lk); + ParallelApplyLedgerKey ttlParallelKey(ttlKey); + auto b = mRoTTLBumps.find(ttlParallelKey); if (b != mRoTTLBumps.end()) { // If we have residual RO TTL bumps for this key, @@ -690,6 +1069,12 @@ ThreadParallelApplyLedgerState::getEntryMap() const return mThreadEntryMap; } +ThreadParallelApplyEntryMap& +ThreadParallelApplyLedgerState::getEntryMap() +{ + return mThreadEntryMap; +} + RestoredEntries const& ThreadParallelApplyLedgerState::getRestoredEntries() const { @@ -699,7 +1084,8 @@ ThreadParallelApplyLedgerState::getRestoredEntries() const ThreadParallelApplyLedgerState::OptionalEntryT ThreadParallelApplyLedgerState::getLiveEntryOpt(LedgerKey const& key) const { - auto it0 = mThreadEntryMap.find(key); + ParallelApplyLedgerKey parallelKey(key); + auto it0 = mThreadEntryMap.find(parallelKey); if (it0 != mThreadEntryMap.end()) { return it0->second.mLedgerEntry; @@ -737,30 +1123,49 @@ ThreadParallelApplyLedgerState::getLiveEntryOpt(LedgerKey const& key) const void ThreadParallelApplyLedgerState::upsertEntry( LedgerKey const& key, ThreadParApplyLedgerEntry const& entry, - uint32_t ledgerSeq) + uint32_t ledgerSeq, bool isNew) { - // Weird syntax avoid extra map lookup auto parAppEntry = ThreadParallelApplyEntry::dirty(entry); parAppEntry.mLedgerEntry.modifyInScope( *this, [&](std::optional& le) { releaseAssertOrThrow(le); le.value().lastModifiedLedgerSeq = ledgerSeq; }); - mThreadEntryMap.insert_or_assign(key, parAppEntry); + // Use try_emplace to preserve mIsNew from the first touch of this entry. + // If the entry already exists in the thread map (from collectCluster or a + // previous TX), keep its mIsNew flag. Otherwise use the caller's isNew. + parAppEntry.mIsNew = isNew; + ParallelApplyLedgerKey parallelKey(key); + auto [it, inserted] = mThreadEntryMap.try_emplace(parallelKey, parAppEntry); + if (!inserted) + { + parAppEntry.mIsNew = it->second.mIsNew; + it->second = parAppEntry; + } } void -ThreadParallelApplyLedgerState::eraseEntry(LedgerKey const& key) +ThreadParallelApplyLedgerState::eraseEntry(LedgerKey const& key, bool isNew) { - auto parAppEntry = ThreadParallelApplyEntry::dirty(scopeAdoptEntryOpt(std::nullopt)); - mThreadEntryMap.insert_or_assign(key, parAppEntry); + // Preserve mIsNew from previous touch, or use caller's isNew for first + // touch. This matters when a subsequent TX recreates the entry: the + // preserved flag determines INIT vs LIVE in commitChangesToLedgerTxn. + parAppEntry.mIsNew = isNew; + ParallelApplyLedgerKey parallelKey(key); + auto [it, inserted] = mThreadEntryMap.try_emplace(parallelKey, parAppEntry); + if (!inserted) + { + parAppEntry.mIsNew = it->second.mIsNew; + it->second = parAppEntry; + } } void ThreadParallelApplyLedgerState::commitChangeFromSuccessfulTx( - LedgerKey const& key, ThreadParApplyLedgerEntryOpt const& newScopedEntryOpt, - UnorderedSet const& roTTLSet) + ParallelApplyLedgerKey const& key, + ThreadParApplyLedgerEntryOpt const& newScopedEntryOpt, + ParallelApplyLedgerKeySet const& roTTLSet) { ThreadParApplyLedgerEntryOpt oldScopedEntryOpt = getLiveEntryOpt(key); std::optional const& oldEntryOpt = @@ -777,12 +1182,16 @@ ThreadParallelApplyLedgerState::commitChangeFromSuccessfulTx( } else if (newEntryOpt) { + // If oldEntryOpt is null, the entry doesn't exist in any parent map + // or persistent state - it's a newly created entry. + bool isNew = !oldEntryOpt.has_value(); upsertEntry(key, scopeAdoptEntry(newEntryOpt.value()), - getSnapshotLedgerSeq() + 1); + getSnapshotLedgerSeq() + 1, isNew); } else { - eraseEntry(key); + bool isNew = !oldEntryOpt.has_value(); + eraseEntry(key, isNew); } } @@ -891,7 +1300,8 @@ TxParallelApplyLedgerState::getLiveEntryOpt(LedgerKey const& key) const // less risky if we don't have to rely on that fact or ensure it in callers: // if callers will get a consistent view of data even if the code changes // and we wind up with some new path calling with a non-empty mTxEntryMap. - auto entryIter = mTxEntryMap.find(key); + ParallelApplyLedgerKey parallelKey(key); + auto entryIter = mTxEntryMap.find(parallelKey); if (entryIter != mTxEntryMap.end()) { return entryIter->second; @@ -903,51 +1313,22 @@ TxParallelApplyLedgerState::getLiveEntryOpt(LedgerKey const& key) const } } -bool +void TxParallelApplyLedgerState::upsertEntry(LedgerKey const& key, LedgerEntry const& entry, uint32_t ledgerSeq) { ZoneScoped; - // There are 4 cases: - // - // 1. The entry exists in the parent maps (thread state or live snapshot) - // but not in mTxEntryMap: we insert it into mTxEntryMap. This is a - // "logical update" even though it's a local insert. We return false. - // - // 2. The entry exists in the parent maps _and_ mTxEntryMap: we update it. - // This is obviously an update! We return false. - // - // 3. The entry does not exist in the parent maps but does already exist in - // mTxEntryMap: we update it. This is a "logical update" to an _earlier_ - // logical create. We return false. - // - // 4. The entry does not exist in the parent maps and does not exist in - // mTxEntryMap: we insert it into mTxEntryMap. This is a "logical - // create". We return true. - // - // The only caller that cares about the return value is a loop that checks - // that logical creates that happened in the soroban host were accompanied - // by logical creates of TTL entries. We could theoretically return true in - // case 3 by comparing against the op prestate rather than the local op - // state, but the only time that happens is when there was a restore that - // populated mTxEntryMap before invoking the host, and we don't especially - // need to check our own TTL-creating work in that case. - - bool liveEntryExistedAlready = - getLiveEntryOpt(key).readInScope(*this).has_value(); - CLOG_TRACE(Tx, "parallel apply thread {} upserting {} key {}", - std::this_thread::get_id(), - liveEntryExistedAlready ? "already-live" : "new", - xdr::xdr_to_string(key, "key")); + CLOG_TRACE(Tx, "parallel apply thread {} upserting key {}", + std::this_thread::get_id(), xdr::xdr_to_string(key, "key")); + ParallelApplyLedgerKey parallelKey(key); auto [mapEntry, _] = - mTxEntryMap.insert_or_assign(key, scopeAdoptEntryOpt(entry)); + mTxEntryMap.insert_or_assign(parallelKey, scopeAdoptEntryOpt(entry)); mapEntry->second.modifyInScope(*this, [&](std::optional& le) { releaseAssertOrThrow(le); le.value().lastModifiedLedgerSeq = ledgerSeq; }); - return !liveEntryExistedAlready; } bool @@ -963,7 +1344,9 @@ TxParallelApplyLedgerState::eraseEntryIfExists(LedgerKey const& key) // any pre-state key when calculating the ledger delta. CLOG_TRACE(Tx, "parallel apply thread {} erasing {}", std::this_thread::get_id(), xdr::xdr_to_string(key, "key")); - mTxEntryMap.insert_or_assign(key, scopeAdoptEntryOpt(std::nullopt)); + ParallelApplyLedgerKey parallelKey(key); + mTxEntryMap.insert_or_assign(parallelKey, + scopeAdoptEntryOpt(std::nullopt)); } else { diff --git a/src/transactions/ParallelApplyUtils.h b/src/transactions/ParallelApplyUtils.h index 73d267e26c..005b393cab 100644 --- a/src/transactions/ParallelApplyUtils.h +++ b/src/transactions/ParallelApplyUtils.h @@ -109,20 +109,20 @@ class ThreadParallelApplyLedgerState // Contains a buffered set of RO TTL bumps that should only be observed // when/if the corresponding entry is modified, otherwise they are merged // (by taking maximums) into the global map at the end of the thread's life. - UnorderedMap mRoTTLBumps; + ParallelApplyLedgerKeyMap mRoTTLBumps; void collectClusterFootprintEntriesFromGlobal( AppConnector& app, GlobalParallelApplyLedgerState const& global, Cluster const& cluster); void upsertEntry(LedgerKey const& key, - ThreadParApplyLedgerEntry const& entry, - uint32_t ledgerSeq); - void eraseEntry(LedgerKey const& key); + ThreadParApplyLedgerEntry const& entry, uint32_t ledgerSeq, + bool isNew = false); + void eraseEntry(LedgerKey const& key, bool isNew = false); void - commitChangeFromSuccessfulTx(LedgerKey const& key, + commitChangeFromSuccessfulTx(ParallelApplyLedgerKey const& key, ThreadParApplyLedgerEntryOpt const& entryOpt, - UnorderedSet const& roTTLSet); + ParallelApplyLedgerKeySet const& roTTLSet); public: ThreadParallelApplyLedgerState(AppConnector& app, @@ -155,6 +155,7 @@ class ThreadParallelApplyLedgerState void flushRemainingRoTTLBumps(); ParallelApplyEntryMap const& getEntryMap() const; + ParallelApplyEntryMap& getEntryMap(); RestoredEntries const& getRestoredEntries() const; @@ -224,22 +225,30 @@ class GlobalParallelApplyLedgerState AppConnector& app, AbstractLedgerTxn& ltx, std::vector const& stages); - bool - maybeMergeRoTTLBumps(LedgerKey const& key, - GlobalParallelApplyEntry const& newEntry, - GlobalParallelApplyEntry& oldEntry, - std::unordered_set const& readWriteSet); - void - commitChangeFromThread(ThreadParallelApplyLedgerState const& thread, - LedgerKey const& key, - ThreadParallelApplyEntry const& parEntry, - std::unordered_set const& readWriteSet); + readOnlyPreParallelApply(AppConnector& app, + std::vector const& txBundles); - void - commitChangesFromThread(AppConnector& app, - ThreadParallelApplyLedgerState const& thread, - std::unordered_set const& readWriteSet); + void commitBufferedPreParallelApplyWrites( + AppConnector& app, AbstractLedgerTxn& ltx, + std::vector const& txBundles); + + void collectModifiedClassicEntries(AbstractLedgerTxn& ltx, + std::vector const& stages); + + bool maybeMergeRoTTLBumps(ParallelApplyLedgerKey const& key, + GlobalParallelApplyEntry const& newEntry, + GlobalParallelApplyEntry& oldEntry, + ParallelApplyLedgerKeySet const& readWriteSet); + + void commitChangeFromThread(ThreadParallelApplyLedgerState const& thread, + ParallelApplyLedgerKey const& key, + ThreadParallelApplyEntry&& parEntry, + ParallelApplyLedgerKeySet const& readWriteSet); + + void commitChangesFromThread(AppConnector& app, + ThreadParallelApplyLedgerState& thread, + ParallelApplyLedgerKeySet const& readWriteSet); public: GlobalParallelApplyLedgerState(AppConnector& app, @@ -258,7 +267,10 @@ class GlobalParallelApplyLedgerState threads, ApplyStage const& stage); - void commitChangesToLedgerTxn(AbstractLedgerTxn& ltx) const; + // Consumes the global entry map: moves entries into the LedgerTxn + // instead of copying. Must only be called once, as the final operation + // on this state (entries are left in a moved-from state afterwards). + void commitChangesToLedgerTxn(AbstractLedgerTxn& ltx); // The snapshot ledger sequence number is one less than the // applying ledger sequence number. @@ -308,7 +320,7 @@ class TxParallelApplyLedgerState // Upsert the entry and sets the lastModifiedLedgerSeq to the given ledger // sequence number. - bool upsertEntry(LedgerKey const& key, LedgerEntry const& entry, + void upsertEntry(LedgerKey const& key, LedgerEntry const& entry, uint32_t ledgerSeq); bool eraseEntryIfExists(LedgerKey const& key); bool entryWasRestored(LedgerKey const& key) const; @@ -330,12 +342,7 @@ class LedgerAccessHelper virtual std::optional getLedgerEntryOpt(LedgerKey const& key) = 0; - // upsert returns true if the entry was created, false if it was updated. - // "created" here is interpreted narrowly to mean there was no - // populated/non-null entry in any parent level of the ledger state; a - // "local" map-insert that shadows an existing entry is not considered a - // create. - virtual bool upsertLedgerEntry(LedgerKey const& key, + virtual void upsertLedgerEntry(LedgerKey const& key, LedgerEntry const& entry) = 0; // erase returns true if the entry was erased, false if it wasn't present. @@ -356,7 +363,7 @@ class PreV23LedgerAccessHelper : virtual public LedgerAccessHelper AbstractLedgerTxn& mLtx; std::optional getLedgerEntryOpt(LedgerKey const& key) override; - bool upsertLedgerEntry(LedgerKey const& key, + void upsertLedgerEntry(LedgerKey const& key, LedgerEntry const& entry) override; bool eraseLedgerEntryIfExists(LedgerKey const& key) override; uint32_t getLedgerVersion() override; @@ -375,7 +382,7 @@ class ParallelLedgerAccessHelper : virtual public LedgerAccessHelper TxParallelApplyLedgerState mTxState; std::optional getLedgerEntryOpt(LedgerKey const& key) override; - bool upsertLedgerEntry(LedgerKey const& key, + void upsertLedgerEntry(LedgerKey const& key, LedgerEntry const& entry) override; bool eraseLedgerEntryIfExists(LedgerKey const& key) override; uint32_t getLedgerVersion() override; diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index 45ce0ec6c9..5c5b2dd48d 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -1897,6 +1897,21 @@ TransactionFrame::checkValidWithOptionallyChargedFee( uint64_t upperBoundCloseTimeOffset, Hash const& envelopeContentsHash, MutableTransactionResultBase& txResult, DiagnosticEventManager& diagnosticEvents) const +{ + checkValidWithOptionallyChargedFee( + app, ls, current, chargeFee, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, envelopeContentsHash, txResult, + diagnosticEvents, nullptr); +} + +void +TransactionFrame::checkValidWithOptionallyChargedFee( + AppConnector& app, LedgerSnapshot const& ls, SequenceNumber current, + bool chargeFee, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, Hash const& envelopeContentsHash, + MutableTransactionResultBase& txResult, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const { ZoneScoped; mCachedAccountPreProtocol8.reset(); @@ -1906,21 +1921,24 @@ TransactionFrame::checkValidWithOptionallyChargedFee( getSignatures(mEnvelope)}; std::optional sorobanResourceFee; - SorobanNetworkConfig const* sorobanConfig = nullptr; + auto effectiveSorobanConfig = sorobanConfig; auto ledgerVersion = ls.getLedgerHeader().current().ledgerVersion; // Load sorobanConfig for all transactions at protocol >= V20. if (protocolVersionStartsFrom(ledgerVersion, SOROBAN_PROTOCOL_VERSION)) { - sorobanConfig = - &app.getLedgerManager().getLastClosedSorobanNetworkConfig(); + if (!effectiveSorobanConfig) + { + effectiveSorobanConfig = + &app.getLedgerManager().getLastClosedSorobanNetworkConfig(); + } if (isSoroban()) { sorobanResourceFee = computePreApplySorobanResourceFee( - ledgerVersion, *sorobanConfig, app.getConfig()); + ledgerVersion, *effectiveSorobanConfig, app.getConfig()); } } - if (commonValid(app, sorobanConfig, signatureChecker, ls, current, false, - chargeFee, lowerBoundCloseTimeOffset, + if (commonValid(app, effectiveSorobanConfig, signatureChecker, ls, current, + false, chargeFee, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, envelopeContentsHash, sorobanResourceFee, txResult, diagnosticEvents) != ValidationType::kMaybeValid) @@ -1933,8 +1951,8 @@ TransactionFrame::checkValidWithOptionallyChargedFee( auto const& op = mOperations[i]; auto& opResult = txResult.getOpResultAt(i); - if (!op->checkValid(app, signatureChecker, sorobanConfig, ls, false, - opResult, diagnosticEvents)) + if (!op->checkValid(app, signatureChecker, effectiveSorobanConfig, ls, + false, opResult, diagnosticEvents)) { // it's OK to just fast fail here and not try to call // checkValid on all operations as the resulting object @@ -1956,6 +1974,18 @@ TransactionFrame::checkValid(AppConnector& app, LedgerSnapshot const& ls, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, DiagnosticEventManager& diagnosticEvents) const +{ + return checkValid(app, ls, current, lowerBoundCloseTimeOffset, + upperBoundCloseTimeOffset, diagnosticEvents, nullptr); +} + +MutableTxResultPtr +TransactionFrame::checkValid(AppConnector& app, LedgerSnapshot const& ls, + SequenceNumber current, + uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const { #ifdef BUILD_TESTS if (app.getRunInOverlayOnlyMode()) @@ -1988,7 +2018,7 @@ TransactionFrame::checkValid(AppConnector& app, LedgerSnapshot const& ls, checkValidWithOptionallyChargedFee( app, ls, current, true, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, getContentsHash(), *txResult, - diagnosticEvents); + diagnosticEvents, sorobanConfig); return txResult; } @@ -2112,6 +2142,111 @@ TransactionFrame::commonPreApply(bool chargeFee, AppConnector& app, } } +std::unique_ptr +TransactionFrame::commonParallelPreApplyReadOnly( + bool chargeFee, AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, + SorobanNetworkConfig const* sorobanConfig, Hash const& envelopeContentsHash, + ParallelPreApplyInfo& info) const +{ + mCachedAccountPreProtocol8.reset(); + uint32_t ledgerVersion = ls.getLedgerHeader().current().ledgerVersion; + std::unique_ptr signatureChecker; +#ifdef BUILD_TESTS + if (txResult.hasReplayTransactionResult()) + { + signatureChecker = std::make_unique( + ledgerVersion, getContentsHash(), getSignatures(mEnvelope)); + } + else + { +#endif // BUILD_TESTS + signatureChecker = std::make_unique( + ledgerVersion, getContentsHash(), getSignatures(mEnvelope)); +#ifdef BUILD_TESTS + } +#endif // BUILD_TESTS + + std::optional sorobanResourceFee; + if (protocolVersionStartsFrom(ledgerVersion, SOROBAN_PROTOCOL_VERSION) && + isSoroban()) + { + sorobanResourceFee = computePreApplySorobanResourceFee( + ledgerVersion, *sorobanConfig, app.getConfig()); + + meta.setNonRefundableResourceFee( + sorobanResourceFee->non_refundable_fee); + int64_t initialFeeRefund = declaredSorobanResourceFee() - + sorobanResourceFee->non_refundable_fee; + txResult.initializeRefundableFeeTracker(initialFeeRefund); + } + + auto cv = + commonValid(app, sorobanConfig, *signatureChecker, ls, 0, true, + chargeFee, 0, 0, envelopeContentsHash, sorobanResourceFee, + txResult, meta.getDiagnosticEventManager()); + info.mUpdateSeqNum = cv >= ValidationType::kInvalidUpdateSeqNum; + + bool signaturesValid = + processSignaturesReadOnly(cv, *signatureChecker, ls, txResult, info); + + if (signaturesValid && cv == ValidationType::kMaybeValid) + { + return signatureChecker; + } + return nullptr; +} + +bool +TransactionFrame::processSignaturesReadOnly( + ValidationType cv, SignatureChecker& signatureChecker, + LedgerSnapshot const& ls, MutableTransactionResultBase& txResult, + ParallelPreApplyInfo& info) const +{ + ZoneScoped; + bool maybeValid = (cv == ValidationType::kMaybeValid); + uint32_t ledgerVersion = ls.getLedgerHeader().current().ledgerVersion; + if (protocolVersionIsBefore(ledgerVersion, ProtocolVersion::V_10)) + { + return maybeValid; + } + + if (protocolVersionStartsFrom(ledgerVersion, ProtocolVersion::V_13) && + !maybeValid) + { + info.mRemoveOneTimeSigners = true; + return false; + } + if (protocolVersionIsBefore(ledgerVersion, ProtocolVersion::V_13) && + cv < ValidationType::kInvalidPostAuth) + { + return false; + } + + bool allOpsValid = true; + if (auto code = txResult.getInnermostResultCode(); + code == txSUCCESS || code == txFAILED) + { + allOpsValid = checkOperationSignatures(signatureChecker, ls, &txResult); + } + + info.mRemoveOneTimeSigners = true; + + if (!allOpsValid) + { + txResult.setInnermostError(txFAILED); + return false; + } + + if (!signatureChecker.checkAllSignaturesUsed()) + { + txResult.setInnermostError(txBAD_AUTH_EXTRA); + return false; + } + + return maybeValid; +} + void TransactionFrame::preParallelApply( AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, @@ -2123,36 +2258,38 @@ TransactionFrame::preParallelApply( } void -TransactionFrame::preParallelApply(bool chargeFee, AppConnector& app, - AbstractLedgerTxn& ltx, - TransactionMetaBuilder& meta, - MutableTransactionResultBase& txResult, - SorobanNetworkConfig const& sorobanConfig, - Hash const& envelopeContentsHash) const +TransactionFrame::preParallelApplyReadOnly( + AppConnector& app, LedgerSnapshot const& ls, TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, ParallelPreApplyInfo& info) const +{ + preParallelApplyReadOnly(true, app, ls, meta, txResult, sorobanConfig, + getContentsHash(), info); +} + +void +TransactionFrame::preParallelApplyReadOnly( + bool chargeFee, AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, Hash const& envelopeContentsHash, + ParallelPreApplyInfo& info) const { ZoneScoped; - releaseAssert(threadIsMain() || - app.threadIsType(Application::ThreadType::APPLY)); try { releaseAssertOrThrow(isSoroban()); - auto signatureChecker = - commonPreApply(chargeFee, app, ltx, meta, txResult, &sorobanConfig, - envelopeContentsHash); + auto signatureChecker = commonParallelPreApplyReadOnly( + chargeFee, app, ls, meta, txResult, &sorobanConfig, + envelopeContentsHash, info); bool ok = signatureChecker != nullptr; if (ok) { - updateSorobanMetrics(app); + info.mUpdateSorobanMetrics = true; auto& opResult = txResult.getOpResultAt(0); - - // Pre parallel soroban, OperationFrame::checkValid is called - // right before OperationFrame::doApply, but we do it here - // instead to avoid making OperationFrame::checkValid thread - // safe. ok = mOperations.front()->checkValid( - app, *signatureChecker, &sorobanConfig, ltx, true, opResult, + app, *signatureChecker, &sorobanConfig, ls, true, opResult, meta.getDiagnosticEventManager()); if (!ok) { @@ -2160,11 +2297,79 @@ TransactionFrame::preParallelApply(bool chargeFee, AppConnector& app, } } - // If validation fails, we check the result code in the parallel - // step to make sure we don't apply the transaction. releaseAssertOrThrow(ok == txResult.isSuccess()); } catch (std::exception& e) + { + printErrorAndAbort("Exception during read-only preParallelApply: ", + e.what()); + } + catch (...) + { + printErrorAndAbort( + "Unknown exception during read-only preParallelApply"); + } +} + +void +TransactionFrame::preParallelApplyWrite(AppConnector& app, + AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const +{ + ZoneScoped; + try + { + LedgerTxn ltxTx(ltx); + if (info.mUpdateSeqNum) + { + processSeqNum(ltxTx); + } + if (info.mRemoveOneTimeSigners) + { + removeOneTimeSignerFromAllSourceAccounts(ltxTx); + } + meta.pushTxChangesBefore(ltxTx); + ltxTx.commit(); + + if (info.mUpdateSorobanMetrics) + { + updateSorobanMetrics(app); + } + } + catch (std::exception& e) + { + printErrorAndAbort("Exception during preParallelApply writes: ", + e.what()); + } + catch (...) + { + printErrorAndAbort("Unknown exception during preParallelApply writes"); + } +} + +void +TransactionFrame::preParallelApply(bool chargeFee, AppConnector& app, + AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + Hash const& envelopeContentsHash) const +{ + ZoneScoped; + releaseAssert(threadIsMain() || + app.threadIsType(Application::ThreadType::APPLY)); + try + { + releaseAssertOrThrow(isSoroban()); + + ParallelPreApplyInfo info; + LedgerSnapshot ls(ltx); + preParallelApplyReadOnly(chargeFee, app, ls, meta, txResult, + sorobanConfig, envelopeContentsHash, info); + preParallelApplyWrite(app, ltx, meta, info); + } + catch (std::exception& e) { printErrorAndAbort("Exception after processing fees but before " "processing sequence number: ", @@ -2230,8 +2435,14 @@ TransactionFrame::parallelApply( if (res) { - threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, - effects); + // Only build the LedgerTxnDelta when invariant checks are + // enabled — the delta is consumed exclusively by + // checkOnOperationApply which is a no-op otherwise. + if (!config.INVARIANT_CHECKS.empty()) + { + threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, + effects); + } opMeta.setLedgerChangesFromSuccessfulOp(threadState, *res, ledgerInfo.getLedgerSeq()); } diff --git a/src/transactions/TransactionFrame.h b/src/transactions/TransactionFrame.h index b73b70cfa5..dddb0ee91b 100644 --- a/src/transactions/TransactionFrame.h +++ b/src/transactions/TransactionFrame.h @@ -245,11 +245,24 @@ class TransactionFrame : public TransactionFrameBase uint64_t upperBoundCloseTimeOffset, Hash const& envelopeContentsHash, MutableTransactionResultBase& result, DiagnosticEventManager& diagnosticEvents) const; + void checkValidWithOptionallyChargedFee( + AppConnector& app, LedgerSnapshot const& ls, SequenceNumber current, + bool chargeFee, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, Hash const& envelopeContentsHash, + MutableTransactionResultBase& result, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const; MutableTxResultPtr checkValid(AppConnector& app, LedgerSnapshot const& ls, SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, DiagnosticEventManager& diagnosticEvents) const override; + MutableTxResultPtr + checkValid(AppConnector& app, LedgerSnapshot const& ls, + SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const override; bool checkSorobanResources( SorobanNetworkConfig const& cfg, uint32_t ledgerVersion, DiagnosticEventManager& diagnosticEvents) const override; @@ -287,18 +300,48 @@ class TransactionFrame : public TransactionFrameBase SorobanNetworkConfig const* sorobanConfig, Hash const& envelopeContentsHash) const; + std::unique_ptr commonParallelPreApplyReadOnly( + bool chargeFee, AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, + SorobanNetworkConfig const* sorobanConfig, + Hash const& envelopeContentsHash, ParallelPreApplyInfo& info) const; + + bool processSignaturesReadOnly(ValidationType cv, + SignatureChecker& signatureChecker, + LedgerSnapshot const& ls, + MutableTransactionResultBase& txResult, + ParallelPreApplyInfo& info) const; + void preParallelApply(bool chargeFee, AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig, Hash const& envelopeContentsHash) const; + void preParallelApplyReadOnly(bool chargeFee, AppConnector& app, + LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + Hash const& envelopeContentsHash, + ParallelPreApplyInfo& info) const; + void preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApplyReadOnly(AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + ParallelPreApplyInfo& info) const override; + + void preParallelApplyWrite(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const override; + std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, diff --git a/src/transactions/TransactionFrameBase.h b/src/transactions/TransactionFrameBase.h index f12f3db2c6..0f69ba186e 100644 --- a/src/transactions/TransactionFrameBase.h +++ b/src/transactions/TransactionFrameBase.h @@ -44,12 +44,65 @@ using TransactionFrameBasePtr = std::shared_ptr; using TransactionFrameBaseConstPtr = std::shared_ptr; +class ParallelApplyLedgerKey +{ + public: + ParallelApplyLedgerKey() = default; + ParallelApplyLedgerKey(LedgerKey const& ledgerKey) : mLedgerKey(ledgerKey) + { + } + + LedgerKey const& + ledgerKey() const + { + return mLedgerKey; + } + + operator LedgerKey const&() const + { + return mLedgerKey; + } + + size_t + hash() const + { + if (mHash != 0) + { + return mHash; + } + mHash = std::hash{}(mLedgerKey); + return mHash; + } + + private: + mutable size_t mHash{0}; + LedgerKey mLedgerKey; +}; + +inline bool +operator==(ParallelApplyLedgerKey const& lhs, ParallelApplyLedgerKey const& rhs) +{ + return lhs.ledgerKey() == rhs.ledgerKey(); +} + +using ParallelApplyLedgerKeySet = UnorderedSet; + +template +using ParallelApplyLedgerKeyMap = UnorderedMap; + // Tracks entry updates within a transaction during parallel apply phases. If // the transaction succeeds, the thread's ParallelApplyEntryMap should be // updated with the entries from the TxModifiedEntryMap. using TxParApplyLedgerEntry = ScopedLedgerEntry; -using TxModifiedEntryMap = UnorderedMap; +using TxModifiedEntryMap = ParallelApplyLedgerKeyMap; + +struct ParallelPreApplyInfo +{ + bool mUpdateSeqNum = false; + bool mRemoveOneTimeSigners = false; + bool mUpdateSorobanMetrics = false; +}; // Used to track the current state of an entry during parallel apply phases. Can // be updated by successful transactions. @@ -59,22 +112,37 @@ template struct ParallelApplyEntry // it due to hitting read limits. ScopedLedgerEntryOpt mLedgerEntry; bool mIsDirty; + // True if this entry was newly created during the parallel apply phase + // (did not exist in persistent state before). Used by + // commitChangesToLedgerTxn to choose createWithoutLoading (INIT) vs + // updateWithoutLoading (LIVE) without expensive existence checks. + bool mIsNew{false}; static ParallelApplyEntry clean(ScopedLedgerEntryOpt const& e) { - return ParallelApplyEntry{e, false}; + return ParallelApplyEntry{e, false, false}; } static ParallelApplyEntry dirty(ScopedLedgerEntryOpt const& e) { - return ParallelApplyEntry{e, true}; + return ParallelApplyEntry{e, true, false}; } template ParallelApplyEntry - rescope(LedgerEntryScope const& s1, LedgerEntryScope const& s2) const + rescope(LedgerEntryScope const& s1, + LedgerEntryScope const& s2) const& { auto adoptedEntry = s2.scopeAdoptEntryOptFrom(mLedgerEntry, s1); - return ParallelApplyEntry{adoptedEntry, mIsDirty}; + return ParallelApplyEntry{adoptedEntry, mIsDirty, mIsNew}; + } + template + ParallelApplyEntry + rescope(LedgerEntryScope const& s1, LedgerEntryScope const& s2) && + { + auto adoptedEntry = + s2.scopeAdoptEntryOptFrom(std::move(mLedgerEntry), s1); + return ParallelApplyEntry{std::move(adoptedEntry), mIsDirty, + mIsNew}; } }; using GlobalParallelApplyEntry = @@ -90,7 +158,7 @@ using TxParallelApplyEntry = // threads return, the updates from each threads entry map should be committed // to LedgerTxn. template -using ParallelApplyEntryMap = UnorderedMap>; +using ParallelApplyEntryMap = ParallelApplyLedgerKeyMap>; using GlobalParallelApplyEntryMap = ParallelApplyEntryMap; using ThreadParallelApplyEntryMap = @@ -162,6 +230,18 @@ class TransactionFrameBase MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig) const = 0; + virtual void + preParallelApplyReadOnly(AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + ParallelPreApplyInfo& info) const = 0; + + virtual void + preParallelApplyWrite(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const = 0; + // If the transaction fails during parallel apply, returns std::nullopt. // Otherwise returns a ParallelTxSuccessVal containing the modified entries // and restored keys. @@ -177,6 +257,15 @@ class TransactionFrameBase SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, DiagnosticEventManager& diagnosticEvents) const = 0; + // Overload that accepts a pre-fetched SorobanNetworkConfig for use in + // parallel validation (where getLedgerManager() cannot be called from + // worker threads due to threadIsMain() assertions). + virtual MutableTxResultPtr + checkValid(AppConnector& app, LedgerSnapshot const& ls, + SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const = 0; virtual bool checkSorobanResources(SorobanNetworkConfig const& cfg, uint32_t ledgerVersion, @@ -290,3 +379,16 @@ class TransactionFrameBase virtual ~TransactionFrameBase() = default; }; } + +namespace std +{ +template <> class hash +{ + public: + size_t + operator()(stellar::ParallelApplyLedgerKey const& key) const + { + return key.hash(); + } +}; +} diff --git a/src/transactions/test/InvokeHostFunctionTests.cpp b/src/transactions/test/InvokeHostFunctionTests.cpp index 74e00bd98f..486ab2c5dc 100644 --- a/src/transactions/test/InvokeHostFunctionTests.cpp +++ b/src/transactions/test/InvokeHostFunctionTests.cpp @@ -53,6 +53,21 @@ using namespace stellar::txtest; namespace { +void +installOneTimeSigner(Application& app, TestAccount& sponsor, + TestAccount& account, SignerKey const& signerKey) +{ + auto signerOp = setOptions(setSigner(Signer{signerKey, 1})); + signerOp.sourceAccount.activate() = toMuxedAccount(account); + + auto signerTx = sponsor.tx({signerOp}); + signerTx->addSignature(account.getSecretKey()); + + auto resultSet = closeLedger(app, {signerTx}); + REQUIRE(resultSet.results.size() == 1); + REQUIRE(isSuccessResult(resultSet.results.front().result)); +} + void checkResults(TransactionResultSet& r, int expectedSuccess, int expectedFailed) { @@ -7901,6 +7916,127 @@ TEST_CASE_VERSIONS("non-fee source account is recipient of payment in both " }); } +TEST_CASE("protocol 26 parallel apply removes soroban pre-auth signer", + "[tx][soroban][parallelapply]") +{ + auto cfg = getTestConfig(); + cfg.LEDGER_PROTOCOL_VERSION = static_cast(ProtocolVersion::V_26); + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = + static_cast(ProtocolVersion::V_26); + + SorobanTest test(cfg, true); + + auto ledgerVersion = getLclProtocolVersion(test.getApp()); + auto startingBalance = + test.getApp().getLedgerManager().getLastMinBalance(50); + + auto source = test.getRoot().create("source", startingBalance); + auto sourceStartingSeq = source.loadSequenceNumber(); + + auto wasm = rust_bridge::get_test_wasm_add_i32(); + auto resources = + defaultUploadWasmResourcesWithoutFootprint(wasm, ledgerVersion); + auto tx = + makeSorobanWasmUploadTx(test.getApp(), source, wasm, resources, 1000); + tx->getMutableEnvelope().v1().signatures.clear(); + + SignerKey txSigner(SIGNER_KEY_TYPE_PRE_AUTH_TX); + txSigner.preAuthTx() = tx->getContentsHash(); + installOneTimeSigner(test.getApp(), test.getRoot(), source, txSigner); + + { + LedgerSnapshot ls(test.getApp()); + auto sourceAccount = ls.load(accountKey(source.getPublicKey())); + REQUIRE(sourceAccount); + REQUIRE(sourceAccount.current().data.account().seqNum == + sourceStartingSeq); + REQUIRE(sourceAccount.current().data.account().signers.size() == 1); + } + + auto r = closeLedger(test.getApp(), {tx}); + REQUIRE(r.results.size() == 1); + checkTx(0, r, txSUCCESS); + + LedgerSnapshot ls(test.getApp()); + auto sourceAccount = ls.load(accountKey(source.getPublicKey())); + REQUIRE(sourceAccount); + REQUIRE(sourceAccount.current().data.account().seqNum == + sourceStartingSeq + 1); + REQUIRE(sourceAccount.current().data.account().signers.empty()); +} + +TEST_CASE("protocol 26 parallel apply removes soroban fee bump pre-auth " + "signers", + "[tx][soroban][parallelapply][feebump]") +{ + auto cfg = getTestConfig(); + cfg.LEDGER_PROTOCOL_VERSION = static_cast(ProtocolVersion::V_26); + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = + static_cast(ProtocolVersion::V_26); + + SorobanTest test(cfg, true); + + auto ledgerVersion = getLclProtocolVersion(test.getApp()); + auto startingBalance = + test.getApp().getLedgerManager().getLastMinBalance(50); + + auto source = test.getRoot().create("source", startingBalance); + auto feeBumper = test.getRoot().create("feeBumper", startingBalance); + auto sourceStartingSeq = source.loadSequenceNumber(); + auto feeBumperStartingSeq = feeBumper.loadSequenceNumber(); + + auto wasm = rust_bridge::get_test_wasm_add_i32(); + auto resources = + defaultUploadWasmResourcesWithoutFootprint(wasm, ledgerVersion); + auto innerTx = + makeSorobanWasmUploadTx(test.getApp(), source, wasm, resources, 1000); + innerTx->getMutableEnvelope().v1().signatures.clear(); + + auto feeBumpTx = feeBump(test.getApp(), feeBumper, innerTx, + innerTx->getEnvelope().v1().tx.fee * 5, + /*useInclusionAsFullFee=*/true); + feeBumpTx->getMutableEnvelope().feeBump().signatures.clear(); + + SignerKey innerSigner(SIGNER_KEY_TYPE_PRE_AUTH_TX); + innerSigner.preAuthTx() = innerTx->getContentsHash(); + installOneTimeSigner(test.getApp(), test.getRoot(), source, innerSigner); + + SignerKey feeBumpSigner(SIGNER_KEY_TYPE_PRE_AUTH_TX); + feeBumpSigner.preAuthTx() = feeBumpTx->getContentsHash(); + installOneTimeSigner(test.getApp(), test.getRoot(), feeBumper, + feeBumpSigner); + + { + LedgerSnapshot ls(test.getApp()); + auto sourceAccount = ls.load(accountKey(source.getPublicKey())); + auto feeBumpAccount = ls.load(accountKey(feeBumper.getPublicKey())); + REQUIRE(sourceAccount); + REQUIRE(feeBumpAccount); + REQUIRE(sourceAccount.current().data.account().seqNum == + sourceStartingSeq); + REQUIRE(feeBumpAccount.current().data.account().seqNum == + feeBumperStartingSeq); + REQUIRE(sourceAccount.current().data.account().signers.size() == 1); + REQUIRE(feeBumpAccount.current().data.account().signers.size() == 1); + } + + auto r = closeLedger(test.getApp(), {feeBumpTx}); + REQUIRE(r.results.size() == 1); + checkTx(0, r, txFEE_BUMP_INNER_SUCCESS); + + LedgerSnapshot ls(test.getApp()); + auto sourceAccount = ls.load(accountKey(source.getPublicKey())); + auto feeBumpAccount = ls.load(accountKey(feeBumper.getPublicKey())); + REQUIRE(sourceAccount); + REQUIRE(feeBumpAccount); + REQUIRE(sourceAccount.current().data.account().seqNum == + sourceStartingSeq + 1); + REQUIRE(feeBumpAccount.current().data.account().seqNum == + feeBumperStartingSeq); + REQUIRE(sourceAccount.current().data.account().signers.empty()); + REQUIRE(feeBumpAccount.current().data.account().signers.empty()); +} + TEST_CASE("parallel txs", "[tx][soroban][parallelapply]") { auto cfg = getTestConfig(); diff --git a/src/transactions/test/StreamingShaTest.cpp b/src/transactions/test/StreamingShaTest.cpp new file mode 100644 index 0000000000..218572fde5 --- /dev/null +++ b/src/transactions/test/StreamingShaTest.cpp @@ -0,0 +1,102 @@ +#include "crypto/ByteSlice.h" +#include "crypto/Hex.h" +#include "crypto/SHA.h" +#include "test/Catch2.h" +#include "test/test.h" +#include "xdr/Stellar-ledger.h" +#include +#include +#include +#include + +using namespace stellar; + +TEST_CASE("Streaming SHA256 for InvokeHostFunctionSuccessPreImage", + "[tx][streaming_sha]") +{ + InvokeHostFunctionSuccessPreImage preImage; + + // 1. Setup returnValue (SCVal) + // Let's make it a simple U32 + preImage.returnValue.type(SCV_U32); + preImage.returnValue.u32() = 0xDEADBEEF; + + // 2. Setup events + // Add a couple of events + ContractEvent event1; + event1.type = DIAGNOSTIC; + event1.body.v0().topics.resize(1); + event1.body.v0().topics[0].type(SCV_SYMBOL); + event1.body.v0().topics[0].sym() = "Topic1"; + event1.body.v0().data.type(SCV_U32); + event1.body.v0().data.u32() = 123; + preImage.events.push_back(event1); + + ContractEvent event2; + event2.type = SYSTEM; + event2.body.v0().topics.resize(2); + event2.body.v0().topics[0].type(SCV_SYMBOL); + event2.body.v0().topics[0].sym() = "Topic2"; + event2.body.v0().topics[1].type(SCV_I32); + event2.body.v0().topics[1].i32() = -42; + event2.body.v0().data.type(SCV_VOID); + preImage.events.push_back(event2); + + // --- Benchmark & Verify xdrSha256 --- + auto start = std::chrono::high_resolution_clock::now(); + Hash hash1 = xdrSha256(preImage); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "xdrSha256 time: " + << std::chrono::duration_cast(end - + start) + .count() + << "ns" << std::endl; + + // --- Prepare Streaming --- + // In the real implementation, we would have raw bytes from the host. + // Here we simulate that by pre-serializing the components. + + xdr::xvector returnValueBytes = + xdr::xdr_to_opaque(preImage.returnValue); + std::vector> eventsBytes; + for (auto const& event : preImage.events) + { + eventsBytes.push_back(xdr::xdr_to_opaque(event)); + } + + // --- Run Streaming SHA256 --- + start = std::chrono::high_resolution_clock::now(); + SHA256 sha; + + // 1. returnValue bytes + sha.add(returnValueBytes); + + // 2. events length (4 bytes big endian) + uint32_t eventsSize = static_cast(preImage.events.size()); + uint32_t eventsSizeBe = + htonl(eventsSize); // Use htonl for network byte order (Big Endian) + sha.add(ByteSlice(reinterpret_cast(&eventsSizeBe), 4)); + + // 3. events bytes + for (auto const& eventBytes : eventsBytes) + { + sha.add(eventBytes); + } + + Hash hash2 = sha.finish(); + end = std::chrono::high_resolution_clock::now(); + std::cout << "Streaming time: " + << std::chrono::duration_cast(end - + start) + .count() + << "ns" << std::endl; + + // --- Verify --- + if (hash1 != hash2) + { + std::cout << "MISMATCH!" << std::endl; + std::cout << "Hash1 (xdrSha256): " << binToHex(hash1) << std::endl; + std::cout << "Hash2 (Streaming): " << binToHex(hash2) << std::endl; + } + REQUIRE(hash1 == hash2); +} diff --git a/src/transactions/test/TransactionTestFrame.cpp b/src/transactions/test/TransactionTestFrame.cpp index 62358e7cae..d5690da5a3 100644 --- a/src/transactions/test/TransactionTestFrame.cpp +++ b/src/transactions/test/TransactionTestFrame.cpp @@ -118,6 +118,19 @@ TransactionTestFrame::checkValid(AppConnector& app, LedgerSnapshot const& ls, return mTransactionTxResult->clone(); } +MutableTxResultPtr +TransactionTestFrame::checkValid( + AppConnector& app, LedgerSnapshot const& ls, SequenceNumber current, + uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const +{ + mTransactionTxResult = mTransactionFrame->checkValid( + app, ls, current, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, + diagnosticEvents, sorobanConfig); + return mTransactionTxResult->clone(); +} + bool TransactionTestFrame::checkValidForTesting(AppConnector& app, AbstractLedgerTxn& ltxOuter, @@ -363,6 +376,24 @@ TransactionTestFrame::preParallelApply( sorobanConfig); } +void +TransactionTestFrame::preParallelApplyReadOnly( + AppConnector& app, LedgerSnapshot const& ls, TransactionMetaBuilder& meta, + MutableTransactionResultBase& resPayload, + SorobanNetworkConfig const& sorobanConfig, ParallelPreApplyInfo& info) const +{ + mTransactionFrame->preParallelApplyReadOnly(app, ls, meta, resPayload, + sorobanConfig, info); +} + +void +TransactionTestFrame::preParallelApplyWrite( + AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const +{ + mTransactionFrame->preParallelApplyWrite(app, ltx, meta, info); +} + std::optional TransactionTestFrame::parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, diff --git a/src/transactions/test/TransactionTestFrame.h b/src/transactions/test/TransactionTestFrame.h index 72f6a451e4..2ef101ba6f 100644 --- a/src/transactions/test/TransactionTestFrame.h +++ b/src/transactions/test/TransactionTestFrame.h @@ -77,6 +77,12 @@ class TransactionTestFrame : public TransactionFrameBase SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, uint64_t upperBoundCloseTimeOffset, DiagnosticEventManager& diagnosticEvents) const override; + MutableTxResultPtr + checkValid(AppConnector& app, LedgerSnapshot const& ls, + SequenceNumber current, uint64_t lowerBoundCloseTimeOffset, + uint64_t upperBoundCloseTimeOffset, + DiagnosticEventManager& diagnosticEvents, + SorobanNetworkConfig const* sorobanConfig) const override; bool checkSorobanResources( SorobanNetworkConfig const& cfg, uint32_t ledgerVersion, DiagnosticEventManager& diagnosticEvents) const override; @@ -151,6 +157,16 @@ class TransactionTestFrame : public TransactionFrameBase MutableTransactionResultBase& resPayload, SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApplyReadOnly(AppConnector& app, LedgerSnapshot const& ls, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& resPayload, + SorobanNetworkConfig const& sorobanConfig, + ParallelPreApplyInfo& info) const override; + + void preParallelApplyWrite(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + ParallelPreApplyInfo const& info) const override; + std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo,