Add Kotlin script for running benchmarks on iOS

benchmarks/multiplatform/README.md

@@ -1,5 +1,48 @@
 # Compose Multiplatform benchmarks
 
+This project contains performance benchmarks for Compose Multiplatform on various targets, 
+including Desktop, iOS, MacOS, and Web (Kotlin/Wasm and Kotlin/JS). 
+These benchmarks measure the performance of various Compose components and features, 
+such as animations, lazy layouts, text rendering, and visual effects.
+
+## Benchmark Modes
+
+The benchmarks can be run in different modes, which determine how performance is measured and reported:
+
+- **`SIMPLE`**: Measures basic frame times without considering VSync. Good for quick checks of raw rendering performance. Enabled by default if no modes are specified.
+- **`VSYNC_EMULATION`**: Emulates VSync behavior to estimate missed frames and provide more realistic performance metrics (CPU/GPU percentiles). Enabled by default if no modes are specified.
+- **`REAL`**: Runs the benchmark in a real-world scenario with actual VSync. This mode provides the most accurate results for user-perceived performance (FPS, actual missed frames)
+              but may not catch performance regressions if a frame fits to budget. 
+              Also requires a device with a real display (may have problems with headless devices). 
+
+To enable specific modes, use the `modes` argument:
+`modes=SIMPLE,VSYNC_EMULATION,REAL`
+
+## Configuration Arguments
+
+You can configure benchmark runs using arguments passed to the Gradle task (via `-PrunArguments="..."`) or to the `.main.kts` script. Arguments can also be set in `gradle.properties` using the `runArguments` property.
+
+| Argument | Description                                                | Example |
+|----------|------------------------------------------------------------|---------|
+| `modes` | Comma-separated list of execution modes (`SIMPLE`, `VSYNC_EMULATION`, `REAL`). | `modes=REAL` |
+| `benchmarks` | Comma-separated list of benchmarks to run. Can optionally specify problem size in parentheses. | `benchmarks=LazyGrid(100),AnimatedVisibility` |
+| `disabledBenchmarks` | Comma-separated list of benchmarks to skip.                | `disabledBenchmarks=HeavyShader` |
+| `warmupCount` | Number of warmup frames before starting measurements.      | `warmupCount=50` |
+| `frameCount` | Number of frames to measure for each benchmark.            | `frameCount=500` |
+| `emptyScreenDelay` | Delay in milliseconds between warmup and measurement (real mode only).| `emptyScreenDelay=1000` |
+| `parallel` | (iOS only) Enable parallel rendering.                      | `parallel=true` |
+| `saveStatsToCSV` | Save results to CSV files.                                 | `saveStatsToCSV=true` |
+| `saveStatsToJSON` | Save results to JSON files.                                | `saveStatsToJSON=true` |
+| `versionInfo` | Add version information to the report.                     | `versionInfo=1.2.3` |
+| `reportAtTheEnd` | Print a summary report after all benchmarks are finished real mode only).| `reportAtTheEnd=true` |
+| `listBenchmarks` | List all available benchmarks and exit.                    | `listBenchmarks=true` |
+
+### Usage Example
+
+```bash
+./gradlew :benchmarks:run -PrunArguments="benchmarks=LazyGrid modes=REAL frameCount=200"
+```
+
 ## Run Desktop
 - `./gradlew :benchmarks:run`
 
@@ -8,13 +51,23 @@ Open the project in Fleet or Android Studio with KMM plugin installed and
 choose `iosApp` run configuration. Make sure that you build the app in `Release` configuration.
 Alternatively you may open `iosApp/iosApp` project in XCode and run the app from there.
 
-## Run automated iOS benchmarks
+## Run iOS benchmarks via scripts
 1. To run on device, open `iosApp/iosApp.xcodeproj` and properly configure the Signing section on the Signing & Capabilities project tab.
 2. Use the following command to get list of all iOS devices:
 - `xcrun xctrace list devices`
 3. From the benchmarks directory run:
-- `./iosApp/run_ios_benchmarks.sh <DEVICE ID>`
-4. Results are saved as `.txt` files in `benchmarks_result/`.
+- `./run_ios_benchmarks.main.kts <DEVICE ID>` (supports all modes of running benchmarks, configured the same way as for other targets:  
+                                               script arguments or `runArguments` property of `gradle.properties`)
+- or `./iosApp/run_ios_benchmarks.sh <DEVICE ID>` (shell script supporting `real` mode benchmarks running with multiple attempts)
+
+To run specific benchmarks:
+- `./run_ios_benchmarks.main.kts <DEVICE ID> benchmarks=AnimatedVisibility,LazyGrid`
+
+To run all benchmarks in a single process (faster but may be less reliable as some 
+benchmarks may affect others within one process):
+- `./run_ios_benchmarks.main.kts <DEVICE ID> separateProcess=false`
+
+4. Results are saved in `benchmarks/build/benchmarks/text-reports/` (when using `.main.kts`) or `benchmarks_result/` (when using `.sh`).
 
 ## Run native on MacOS
  - `./gradlew :benchmarks:runReleaseExecutableMacosArm64` (Works on Arm64 processors)

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt

@@ -327,6 +327,11 @@ suspend fun runBenchmark(
             content = benchmark.content
         ).generateStats()
         stats.prettyPrint()
+        if (Config.saveStatsToJSON && isIosTarget) {
+            println("JSON_START")
+            println(stats.toJsonString())
+            println("JSON_END")
+        }
         if (Config.saveStats()) {
             saveBenchmarkStats(name = benchmark.name, stats = stats)
         }
@@ -341,6 +346,12 @@ suspend fun runBenchmarks(
     warmupCount: Int = Config.warmupCount,
     graphicsContext: GraphicsContext? = null
 ) {
+    if (Config.listBenchmarks) {
+        println("AVAILABLE_BENCHMARKS_START")
+        benchmarks.forEach { println(it.name) }
+        println("AVAILABLE_BENCHMARKS_END")
+        return
+    }
     println()
     println("Running emulating $targetFps FPS")
     println()
@@ -419,6 +430,11 @@ fun BenchmarkRunner(
                     } else {
                         results.add(stats)
                     }
+                    if (Config.saveStatsToJSON && isIosTarget) {
+                        println("JSON_START")
+                        println(stats.toJsonString())
+                        println("JSON_END")
+                    }
                     if (Config.saveStats()) {
                         saveBenchmarkStats(name = benchmark.name, stats = stats)
                     }

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt

@@ -70,6 +70,8 @@ fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) {
  */
 expect fun saveBenchmarkStats(name: String, stats: BenchmarkStats)
 
+expect val isIosTarget: Boolean
+
 private fun RawSource.readText() = use {
     it.buffered().readByteArray().decodeToString()
 }

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt

@@ -27,7 +27,7 @@ object Args {
      * @param args an array of strings representing the command line arguments.
      * Each argument can specify either of these settings:
      * modes, benchmarks, disabledBenchmarks - comma separated values,
-     * versionInfo, saveStatsToCSV, saveStatsToJSON, parallel, warmupCount, frameCount, emptyScreenDelay, reportAtTheEnd - single values.
+     * versionInfo, saveStatsToCSV, saveStatsToJSON, parallel, warmupCount, frameCount, emptyScreenDelay, reportAtTheEnd, listBenchmarks - single values.
      *
      * Example: benchmarks=AnimatedVisibility(100),modes=SIMPLE,versionInfo=Kotlin_2_1_20,saveStatsToCSV=true,warmupCount=50,frameCount=100,emptyScreenDelay=2000,reportAtTheEnd=true
      */
@@ -44,6 +44,7 @@ object Args {
         var frameCount: Int? = null
         var emptyScreenDelay: Long? = null
         var reportAtTheEnd: Boolean = false
+        var listBenchmarks: Boolean = false
 
         for (arg in args) {
             if (arg.startsWith("modes=", ignoreCase = true)) {
@@ -70,6 +71,8 @@ object Args {
                 emptyScreenDelay = arg.substringAfter("=").toLong()
             } else if (arg.startsWith("reportAtTheEnd=", ignoreCase = true)) {
                 reportAtTheEnd = arg.substringAfter("=").toBoolean()
+            } else if (arg.startsWith("listBenchmarks=", ignoreCase = true)) {
+                listBenchmarks = arg.substringAfter("=").toBoolean()
             } else {
                 println("WARNING: unknown argument $arg")
             }
@@ -89,7 +92,8 @@ object Args {
             warmupCount = warmupCount ?: defaultWarmupCount,
             frameCount = frameCount ?: 1000,
             emptyScreenDelay = emptyScreenDelay ?: 2000L,
-            reportAtTheEnd = reportAtTheEnd
+            reportAtTheEnd = reportAtTheEnd,
+            listBenchmarks = listBenchmarks
         )
     }
 }
@@ -110,6 +114,7 @@ object Args {
  * @property frameCount Number of frames to run for each benchmark.
  * @property emptyScreenDelay Delay in milliseconds between warmup and benchmark.
  * @property reportAtTheEnd Flag indicating whether we should report results at the end of all benchmarks.
+ * @property listBenchmarks Flag indicating whether we should print available benchmarks and exit.
  */
 data class Config(
     val modes: Set<Mode> = emptySet(),
@@ -123,7 +128,8 @@ data class Config(
     val warmupCount: Int = 100,
     val frameCount: Int = 1000,
     val emptyScreenDelay: Long = 2000L,
-    val reportAtTheEnd: Boolean = false
+    val reportAtTheEnd: Boolean = false,
+    val listBenchmarks: Boolean = false
 ) {
     /**
      * Checks if a specific mode is enabled based on the configuration.
@@ -185,6 +191,9 @@ data class Config(
         val reportAtTheEnd: Boolean
             get() = global.reportAtTheEnd
 
+        val listBenchmarks: Boolean
+            get() = global.listBenchmarks
+
         fun setGlobal(global: Config) {
             this.global = global
         }

benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt

@@ -4,3 +4,5 @@
  */
 
 actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) = saveBenchmarkStatsOnDisk(name, stats)
+
+actual val isIosTarget: Boolean = false

benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt

@@ -7,6 +7,7 @@ import androidx.compose.ui.ExperimentalComposeUiApi
 import androidx.compose.ui.window.ComposeUIViewController
 import kotlinx.coroutines.MainScope
 import kotlinx.coroutines.launch
+import platform.UIKit.UIApplication
 import platform.UIKit.UIScreen
 import platform.UIKit.UIViewController
 import kotlin.system.exitProcess
@@ -18,12 +19,16 @@ fun setGlobalFromArgs(args : List<String>) {
 fun runReal() = Config.isModeEnabled(Mode.REAL)
 
 fun runBenchmarks() {
+    UIApplication.sharedApplication.setIdleTimerDisabled(true)
     MainScope().launch {
         runBenchmarks(graphicsContext = graphicsContext())
         println("Completed!")
+        exitProcess(0)
     }
 }
 
+actual val isIosTarget: Boolean = true
+
 @OptIn(ExperimentalComposeUiApi::class)
 fun MainViewController(): UIViewController {
     return ComposeUIViewController(configure = { parallelRendering = Config.parallelRendering }) {

benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt

@@ -12,6 +12,8 @@ import platform.AppKit.NSScreen
 import platform.AppKit.maximumFramesPerSecond
 import kotlin.system.exitProcess
 
+actual val isIosTarget: Boolean = true
+
 fun main(args : Array<String>) {
     Config.setGlobalFromArgs(args)
     if (Config.isModeEnabled(Mode.REAL)) {

benchmarks/multiplatform/benchmarks/src/webMain/kotlin/BenchmarksSave.web.kt

@@ -30,6 +30,8 @@ actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) {
     }
 }
 
+actual val isIosTarget: Boolean = false
+
 /**
  * Client for sending benchmark results to the server
  */

benchmarks/multiplatform/compare_benchmarks.main.kts

@@ -17,7 +17,7 @@ fun main(args: Array<String>) {
     val v2 = argMap["v2"] ?: args.getOrNull(1)
 
     if (v1 == null || v2 == null) {
-        println("Usage: compare_benchmarks.main.kts v1=<version1> v2=<version2> [runs=3] [benchmarks=<benchmarkName>] [platform=macos|desktop|web]")
+        println("Usage: ./compare_benchmarks.main.kts v1=<version1> v2=<version2> [runs=3] [benchmarks=<benchmarkName>] [platform=macos|desktop|web|ios]")
         return
     }
 
@@ -26,28 +26,29 @@ fun main(args: Array<String>) {
     val platform = argMap["platform"] ?: "macos"
     val runServer = platform == "web"
     val isWeb = platform == "web"
+    val isIos = platform == "ios"
 
     println("Comparing Compose versions: $v1 and $v2")
     println("Number of runs: $runs")
     println("Platform: $platform")
     benchmarkName?.let { println("Filtering by benchmark: $it") }
 
-    val resultsV1 = runBenchmarksForVersion(v1, runs, benchmarkName, platform, runServer, isWeb)
-    val resultsV2 = runBenchmarksForVersion(v2, runs, benchmarkName, platform, runServer, isWeb)
+    val resultsV1 = runBenchmarksForVersion(v1, runs, benchmarkName, platform, runServer, isWeb, isIos)
+    val resultsV2 = runBenchmarksForVersion(v2, runs, benchmarkName, platform, runServer, isWeb, isIos)
 
     compareResults(v1, resultsV1, v2, resultsV2)
 }
 
 data class BenchmarkResult(val name: String, val totalMs: Double)
 
-fun runBenchmarksForVersion(version: String, runs: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean): Map<String, List<Double>> {
+fun runBenchmarksForVersion(version: String, runs: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean, isIos: Boolean): Map<String, List<Double>> {
     println("\n=== Running benchmarks for version: $version ===")
 
     val allRunsResults = mutableMapOf<String, MutableList<Double>>()
 
     for (i in 1..runs) {
         println("Run $i/$runs...")
-        executeBenchmarks(version, i, benchmarkName, platform, runServer, isWeb)
+        executeBenchmarks(version, i, benchmarkName, platform, runServer, isWeb, isIos)
         val runResults = collectResults(version, i)
         runResults.forEach { (name, value) ->
             allRunsResults.getOrPut(name) { mutableListOf() }.add(value)
@@ -65,7 +66,7 @@ fun updateComposeVersion(version: String) {
     println("Updated gradle/libs.versions.toml to version $version")
 }
 
-fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean) {
+fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean, isIos: Boolean) {
     val (versionedExecutable, defaultExecutable, task) = when (platform) {
         "macos" -> Triple(
             File("benchmarks/build/bin/macosArm64/releaseExecutable/benchmarks-$version.kexe"),
@@ -82,6 +83,11 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl
             null,
             ":benchmarks:wasmJsBrowserProductionRun"
         )
+        "ios" -> Triple(
+            null,
+            null,
+            "ios"
+        )
         else -> throw IllegalArgumentException("Unsupported platform: $platform")
     }
 
@@ -131,7 +137,7 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl
         Thread.sleep(5000)
 
         try {
-            executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, serverStopped)
+            executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, isIos, serverStopped)
         } finally {
             println("Stopping benchmark server...")
             serverProcess.destroy()
@@ -140,7 +146,7 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl
             monitorThread.interrupt()
         }
     } else {
-        executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, null)
+        executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, isIos, null)
     }
 }
 
@@ -152,8 +158,24 @@ fun executeBenchmarksOnce(
     versionedExecutable: File?,
     defaultExecutable: File?,
     isWeb: Boolean,
+    isIos: Boolean,
     serverStopped: java.util.concurrent.atomic.AtomicBoolean?
 ) {
+    if (isIos) {
+        println("Running version $version on iOS...")
+        updateComposeVersion(version)
+        val processBuilder = ProcessBuilder(
+            "./run_ios_benchmarks.main.kts",
+            *runArgs.toTypedArray()
+        ).inheritIO()
+        val process = processBuilder.start()
+        val exitCode = process.waitFor()
+        if (exitCode != 0) {
+            println("Warning: iOS Benchmark run failed with exit code $exitCode")
+        }
+        return
+    }
+
     if (versionedExecutable != null && versionedExecutable.exists()) {
         println("Using existing executable for version $version: ${versionedExecutable.absolutePath}")
 

benchmarks/multiplatform/iosApp/run_ios_benchmarks.sh

@@ -32,21 +32,6 @@ CONFIGURATION="Release"
 ATTEMPTS=5
 BUILD_DIR="$MULTIPLATFORM_DIR/.benchmark_build"
 
-BENCHMARKS=(
-    "AnimatedVisibility"
-    "LazyGrid"
-    "LazyGrid-ItemLaunchedEffect"
-    "LazyGrid-SmoothScroll"
-    "LazyGrid-SmoothScroll-ItemLaunchedEffect"
-    "VisualEffects"
-    "LazyList"
-    "MultipleComponents"
-    "MultipleComponents-NoVectorGraphics"
-    "TextLayout"
-    "CanvasDrawing"
-    "HeavyShader"
-)
-
 # ── Helpers ────────────────────────────────────────────────────────────────────
 
 die() { echo ""; echo "ERROR: $*" >&2; exit 1; }
@@ -192,6 +177,18 @@ mkdir -p "$OUTPUT_DIR"
 
 # ── 4. Run benchmarks ──────────────────────────────────────────────────────────
 
+echo ""
+echo "==> [4/4] Fetching benchmarks list via Gradle..."
+TEMP_LIST=$(mktemp)
+(cd "$MULTIPLATFORM_DIR" && ./gradlew :benchmarks:run -PrunArguments=listBenchmarks=true) > "$TEMP_LIST" 2>&1
+
+BENCHMARKS=($(awk '/AVAILABLE_BENCHMARKS_START/{p=1;next} /AVAILABLE_BENCHMARKS_END/{p=0} p && NF{print}' "$TEMP_LIST"))
+rm "$TEMP_LIST"
+
+if [[ ${#BENCHMARKS[@]} -eq 0 ]]; then
+    die "No benchmarks found to run."
+fi
+
 TOTAL=$(( ${#BENCHMARKS[@]} * 2 * ATTEMPTS ))
 CURRENT=0