From b70e92009be209aaae95df098d66cfc07314b160 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 15:26:52 +0200 Subject: [PATCH 01/10] run_ios_benchmarks.sh was rewritten in Kotlin to run_ios_benchmarks.main.kts --- .../multiplatform/run_ios_benchmarks.main.kts | 332 ++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100755 benchmarks/multiplatform/run_ios_benchmarks.main.kts diff --git a/benchmarks/multiplatform/run_ios_benchmarks.main.kts b/benchmarks/multiplatform/run_ios_benchmarks.main.kts new file mode 100755 index 0000000000..4b137822f9 --- /dev/null +++ b/benchmarks/multiplatform/run_ios_benchmarks.main.kts @@ -0,0 +1,332 @@ +#!/usr/bin/env kotlin + +import java.io.File +import java.io.InputStream +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * run_ios_benchmarks.main.kts + * + * Builds the iosApp, installs it on a real device or simulator, then runs + * every benchmark (from Benchmarks.kt) with parallel=true and parallel=false, + * ATTEMPTS times each. Console output is saved to: + * + * benchmarks_result/__parallel___.txt + * + * Requirements: + * - Xcode 15+ (uses xcrun devicectl for real devices, xcrun simctl for simulators) + * - For real device: connected via USB and trusted, valid code-signing identity + * - For simulator: any booted or available simulator + * + * Usage: ./run_ios_benchmarks.main.kts [] + * + * If no UDID is provided the first connected real device is used. + * Pass a simulator UDID to target a simulator instead. + */ + +// ── Configuration ────────────────────────────────────────────────────────────── + +val ROOT_DIR = File(".").absoluteFile +val PROJECT_DIR = File(ROOT_DIR, "iosApp") +val MULTIPLATFORM_DIR = ROOT_DIR +val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "benchmarks_result") +val SCHEME = "iosApp" +val CONFIGURATION = "Release" +val ATTEMPTS = 5 +val BUILD_DIR = File(MULTIPLATFORM_DIR, ".benchmark_build") + +val BENCHMARKS = listOf( + "AnimatedVisibility", + "LazyGrid", + "LazyGrid-ItemLaunchedEffect", + "LazyGrid-SmoothScroll", + "LazyGrid-SmoothScroll-ItemLaunchedEffect", + "VisualEffects", + "LazyList", + "MultipleComponents", + "MultipleComponents-NoVectorGraphics", + "TextLayout", + "CanvasDrawing", + "HeavyShader" +) + +// ── Helpers ──────────────────────────────────────────────────────────────────── + +fun die(message: String): Nothing { + System.err.println("\nERROR: $message") + System.exit(1) + throw RuntimeException(message) +} + +fun exec(vararg command: String, workingDir: File = ROOT_DIR, redirectStderr: Boolean = true): String { + val process = ProcessBuilder(*command) + .directory(workingDir) + .redirectErrorStream(redirectStderr) + .start() + + val output = process.inputStream.bufferedReader().readText() + val exitCode = process.waitFor() + if (exitCode != 0) { + if (!redirectStderr) { + val error = process.errorStream.bufferedReader().readText() + System.err.println(error) + } + println(output) + die("Command failed with exit code $exitCode: ${command.joinToString(" ")}") + } + return output +} + +fun execInheritIO(vararg command: String, workingDir: File = ROOT_DIR) { + val process = ProcessBuilder(*command) + .directory(workingDir) + .inheritIO() + .start() + + val exitCode = process.waitFor() + if (exitCode != 0) { + die("Command failed with exit code $exitCode: ${command.joinToString(" ")}") + } +} + +fun execWithTee(command: List, outputFile: File, workingDir: File = ROOT_DIR): Int { + val process = ProcessBuilder(command) + .directory(workingDir) + .redirectErrorStream(true) + .start() + + val reader = process.inputStream.bufferedReader() + outputFile.parentFile.mkdirs() + outputFile.bufferedWriter().use { writer -> + var line: String? = reader.readLine() + while (line != null) { + println(line) + writer.write(line) + writer.newLine() + line = reader.readLine() + } + } + + return process.waitFor() +} + +// ── 1. Detect target device or simulator ────────────────────────────────────── + +println("\n==> [1/4] Detecting target...") + +println(" $ xcrun xctrace list devices") +val xctraceOut = try { + exec("xcrun", "xctrace", "list", "devices") +} catch (e: Exception) { + die("Failed to run xctrace: ${e.message}") +} + +fun parseDevices(output: String, sectionName: String): List { + val lines = output.lines() + val result = mutableListOf() + var inSection = false + for (line in lines) { + if (line.startsWith("== $sectionName ==")) { + inSection = true + continue + } + if (line.startsWith("== ")) { + inSection = false + continue + } + if (inSection && line.isNotBlank()) { + // Filter out Mac and ensure it has a version number + if (!line.contains(" Mac ") && line.contains(Regex("""\(\d+\.\d+"""))) { + result.add(line.trim()) + } + } + } + return result +} + +val allRealLines = parseDevices(xctraceOut, "Devices") +val allSimLines = parseDevices(xctraceOut, "Simulators") +val allDeviceLines = allRealLines + allSimLines + +if (allDeviceLines.isEmpty()) { + println(xctraceOut) + die("No iOS device or simulator found.") +} + +val argUdid = args.getOrNull(0) +val deviceLine = if (argUdid != null) { + allDeviceLines.find { it.contains(argUdid) } ?: run { + println("Available devices and simulators:") + allDeviceLines.forEach { println(it) } + die("Device with UDID '$argUdid' not found among the above.") + } +} else { + if (allRealLines.isNotEmpty()) allRealLines.first() else allSimLines.first() +} + +val isSimulator = allSimLines.contains(deviceLine) + +// Parse "Device Name (iOS Version) (UDID)" +// UDID is the last parenthesised token on the line. +val udidRegex = Regex("""\(([0-9A-Fa-f-]+)\)""") +val deviceId = udidRegex.findAll(deviceLine).lastOrNull()?.groupValues?.get(1) ?: die("Could not parse UDID from: $deviceLine") + +val versionRegex = Regex("""\(([0-9]+\.[0-9.]+)\)""") +val deviceIos = versionRegex.findAll(deviceLine).firstOrNull()?.groupValues?.get(1) ?: die("Could not parse iOS version from: $deviceLine") + +val deviceName = deviceLine.substringBefore(" ($deviceIos").trim() + +// Normalize for filenames: lowercase, spaces→underscores, keep only [a-z0-9._-] +val devicePrefix = "${deviceName}_$deviceIos" + .lowercase() + .replace(" ", "_") + .filter { it in 'a'..'z' || it in '0'..'9' || it == '.' || it == '_' || it == '-' } + +println(" Name : $deviceName") +println(" iOS : $deviceIos") +println(" UDID : $deviceId") +println(" Simulator : $isSimulator") +println(" Prefix : ${devicePrefix}_parallel___.txt") + +// ── 2. Build ─────────────────────────────────────────────────────────────────── + +println("\n==> [2/4] Building '$SCHEME' ($CONFIGURATION)...") +println(" $ mkdir -p $BUILD_DIR") +BUILD_DIR.mkdirs() + +val xcodeLog = File(BUILD_DIR, "xcodebuild.log") + +// Clean stale Kotlin Native build artifacts to avoid klib ABI version mismatches. +println(" $ ./gradlew clean") +execInheritIO("./gradlew", "clean") + +println(" $ xcodebuild build -project ${PROJECT_DIR.path}/iosApp.xcodeproj -scheme $SCHEME -configuration $CONFIGURATION -destination id=$deviceId ONLY_ACTIVE_ARCH=YES SYMROOT=${BUILD_DIR.path}") + +val xcodebuildProcess = ProcessBuilder( + "xcodebuild", "build", + "-project", "${PROJECT_DIR.path}/iosApp.xcodeproj", + "-scheme", SCHEME, + "-configuration", CONFIGURATION, + "-destination", "id=$deviceId", + "ONLY_ACTIVE_ARCH=YES", + "SYMROOT=${BUILD_DIR.path}" +).redirectErrorStream(true).start() + +xcodeLog.bufferedWriter().use { writer -> + xcodebuildProcess.inputStream.bufferedReader().forEachLine { line -> + writer.write(line) + writer.newLine() + } +} +val buildExit = xcodebuildProcess.waitFor() + +if (buildExit != 0) { + println("Build failed. Last 50 lines of xcodebuild output:") + println("----------------------------------------------------") + val logLines = xcodeLog.readLines() + logLines.takeLast(50).forEach { println(it) } + println("----------------------------------------------------") + println("Full log: ${xcodeLog.path}") + System.exit(1) +} + +val appPath = if (isSimulator) { + File(BUILD_DIR, "${CONFIGURATION}-iphonesimulator/ComposeBenchmarks.app") +} else { + File(BUILD_DIR, "${CONFIGURATION}-iphoneos/ComposeBenchmarks.app") +} + +if (!appPath.exists()) die("App bundle not found at expected path: ${appPath.path}") + +println(" $ /usr/libexec/PlistBuddy -c 'Print CFBundleIdentifier' ${appPath.path}/Info.plist") +val bundleId = exec("/usr/libexec/PlistBuddy", "-c", "Print CFBundleIdentifier", "${appPath.path}/Info.plist").trim() +println(" Build : OK") +println(" Bundle : $bundleId") + +// ── 3. Install ───────────────────────────────────────────────────────────────── + +println("\n==> [3/4] Installing...") + +if (isSimulator) { + // Boot the simulator if it is not already running. + val simctlList = exec("xcrun", "simctl", "list", "devices") + val simStateLine = simctlList.lines().find { it.contains(deviceId) } + val isBooted = simStateLine?.contains("(Booted)") == true + + if (!isBooted) { + println(" $ xcrun simctl boot $deviceId") + exec("xcrun", "simctl", "boot", deviceId) + } + println(" $ xcrun simctl install $deviceId ${appPath.path}") + exec("xcrun", "simctl", "install", deviceId, appPath.path) +} else { + println(" $ xcrun devicectl device install app --device $deviceId ${appPath.path}") + exec("xcrun", "devicectl", "device", "install", "app", "--device", deviceId, appPath.path) +} +println(" Installed.") + +println(" $ mkdir -p $OUTPUT_DIR") +OUTPUT_DIR.mkdirs() + +// ── 4. Run benchmarks ────────────────────────────────────────────────────────── + +val total = BENCHMARKS.size * 2 * ATTEMPTS +var current = 0 + +println("\n==> [4/4] Running $total benchmark sessions") +println(" ${BENCHMARKS.size} benchmarks × 2 parallel modes × $ATTEMPTS attempts\n") + +for (benchmark in BENCHMARKS) { + for (parallel in listOf("true", "false")) { + for (attempt in 1..ATTEMPTS) { + current++ + + val outFileName = "${devicePrefix}_parallel_${parallel}_${benchmark}_${attempt}.txt" + val outFile = File(OUTPUT_DIR, outFileName) + + System.out.format(" [%3d/%3d] %-52s parallel=%-5s attempt=%d\n", + current, total, benchmark, parallel, attempt) + println(" → ${outFile.name}") + + val runArgs = mutableListOf() + val exitCode = if (isSimulator) { + runArgs.addAll(listOf( + "xcrun", "simctl", "launch", "--console", deviceId, bundleId, + "benchmarks=$benchmark", + "parallel=$parallel", + "warmupCount=100", + "modes=REAL", + "reportAtTheEnd=true" + )) + println(" $ ${runArgs.joinToString(" ")}") + execWithTee(runArgs, outFile) + } else { + runArgs.addAll(listOf( + "xcrun", "devicectl", "device", "process", "launch", "--console", "--device", deviceId, bundleId, + "--", + "benchmarks=$benchmark", + "parallel=$parallel", + "warmupCount=100", + "modes=REAL", + "reportAtTheEnd=true" + )) + println(" $ ${runArgs.joinToString(" ")}") + execWithTee(runArgs, outFile) + } + + if (exitCode != 0) { + System.out.format(" ⚠ WARNING: process exited with code %d\n", exitCode) + } else { + println(" ✓ done") + } + + // Brief cooldown between runs so the device settles + println(" $ sleep 3") + Thread.sleep(3000) + } + } +} + +println("\n==> All done!") +System.out.format(" %d output files written to: %s\n\n", total, OUTPUT_DIR.path) From 48033a36633543699b6baac704761149141be080 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 18:37:53 +0200 Subject: [PATCH 02/10] Make arguments the same as runArguments gradle properties and introduce separateProcess flag for running benchmarks in separate process --- .../multiplatform/run_ios_benchmarks.main.kts | 168 +++++++++++------- 1 file changed, 105 insertions(+), 63 deletions(-) diff --git a/benchmarks/multiplatform/run_ios_benchmarks.main.kts b/benchmarks/multiplatform/run_ios_benchmarks.main.kts index 4b137822f9..22f0e9893a 100755 --- a/benchmarks/multiplatform/run_ios_benchmarks.main.kts +++ b/benchmarks/multiplatform/run_ios_benchmarks.main.kts @@ -9,10 +9,10 @@ import java.util.concurrent.TimeUnit * run_ios_benchmarks.main.kts * * Builds the iosApp, installs it on a real device or simulator, then runs - * every benchmark (from Benchmarks.kt) with parallel=true and parallel=false, - * ATTEMPTS times each. Console output is saved to: + * every benchmark (from Benchmarks.kt). + * Console output is saved to: * - * benchmarks_result/__parallel___.txt + * benchmarks_result/__.txt * * Requirements: * - Xcode 15+ (uses xcrun devicectl for real devices, xcrun simctl for simulators) @@ -33,10 +33,9 @@ val MULTIPLATFORM_DIR = ROOT_DIR val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "benchmarks_result") val SCHEME = "iosApp" val CONFIGURATION = "Release" -val ATTEMPTS = 5 val BUILD_DIR = File(MULTIPLATFORM_DIR, ".benchmark_build") -val BENCHMARKS = listOf( +val DEFAULT_BENCHMARKS = listOf( "AnimatedVisibility", "LazyGrid", "LazyGrid-ItemLaunchedEffect", @@ -51,6 +50,39 @@ val BENCHMARKS = listOf( "HeavyShader" ) +// ── Argument Parsing ────────────────────────────────────────────────────────── + +val parsedArgs = mutableMapOf() +val rawArgs = if (args.isEmpty() || (args.size == 1 && !args[0].contains("="))) { + val gradleProps = File(ROOT_DIR, "gradle.properties") + if (gradleProps.exists()) { + val runArgsLine = gradleProps.readLines().find { it.startsWith("runArguments=") } + runArgsLine?.substringAfter("runArguments=")?.split(" ")?.filter { it.isNotBlank() } ?: emptyList() + } else { + emptyList() + } +} else { + args.toList() +} + +// UDID can be passed as a standalone first argument if it doesn't contain '=' +val argUdidFromArgs = if (args.isNotEmpty() && !args[0].contains("=") && !args[0].startsWith("-")) args[0] else null + +rawArgs.forEach { arg -> + if (arg.contains("=")) { + val (key, value) = arg.split("=", limit = 2) + parsedArgs[key.lowercase()] = value + } +} + +val benchmarksFromArgs = parsedArgs["benchmarks"]?.split(",")?.map { it.substringBefore("(").trim() }?.filter { it.isNotEmpty() } +val benchmarksToRun = benchmarksFromArgs ?: DEFAULT_BENCHMARKS +val separateProcess = parsedArgs["separateprocess"]?.toBoolean() ?: true + +// Arguments to pass to the app +val appArgs = parsedArgs.toMutableMap() +appArgs.remove("separateprocess") + // ── Helpers ──────────────────────────────────────────────────────────────────── fun die(message: String): Nothing { @@ -154,12 +186,11 @@ if (allDeviceLines.isEmpty()) { die("No iOS device or simulator found.") } -val argUdid = args.getOrNull(0) -val deviceLine = if (argUdid != null) { - allDeviceLines.find { it.contains(argUdid) } ?: run { +val deviceLine = if (argUdidFromArgs != null) { + allDeviceLines.find { it.contains(argUdidFromArgs) } ?: run { println("Available devices and simulators:") allDeviceLines.forEach { println(it) } - die("Device with UDID '$argUdid' not found among the above.") + die("Device with UDID '$argUdidFromArgs' not found among the above.") } } else { if (allRealLines.isNotEmpty()) allRealLines.first() else allSimLines.first() @@ -187,7 +218,7 @@ println(" Name : $deviceName") println(" iOS : $deviceIos") println(" UDID : $deviceId") println(" Simulator : $isSimulator") -println(" Prefix : ${devicePrefix}_parallel___.txt") +println(" Prefix : ${devicePrefix}_.txt") // ── 2. Build ─────────────────────────────────────────────────────────────────── @@ -198,9 +229,9 @@ BUILD_DIR.mkdirs() val xcodeLog = File(BUILD_DIR, "xcodebuild.log") // Clean stale Kotlin Native build artifacts to avoid klib ABI version mismatches. -println(" $ ./gradlew clean") -execInheritIO("./gradlew", "clean") - +//println(" $ ./gradlew clean") +//execInheritIO("./gradlew", "clean") +// println(" $ xcodebuild build -project ${PROJECT_DIR.path}/iosApp.xcodeproj -scheme $SCHEME -configuration $CONFIGURATION -destination id=$deviceId ONLY_ACTIVE_ARCH=YES SYMROOT=${BUILD_DIR.path}") val xcodebuildProcess = ProcessBuilder( @@ -271,62 +302,73 @@ OUTPUT_DIR.mkdirs() // ── 4. Run benchmarks ────────────────────────────────────────────────────────── -val total = BENCHMARKS.size * 2 * ATTEMPTS -var current = 0 +val total = if (separateProcess) benchmarksToRun.size else 1 println("\n==> [4/4] Running $total benchmark sessions") -println(" ${BENCHMARKS.size} benchmarks × 2 parallel modes × $ATTEMPTS attempts\n") - -for (benchmark in BENCHMARKS) { - for (parallel in listOf("true", "false")) { - for (attempt in 1..ATTEMPTS) { - current++ - - val outFileName = "${devicePrefix}_parallel_${parallel}_${benchmark}_${attempt}.txt" - val outFile = File(OUTPUT_DIR, outFileName) - - System.out.format(" [%3d/%3d] %-52s parallel=%-5s attempt=%d\n", - current, total, benchmark, parallel, attempt) - println(" → ${outFile.name}") - - val runArgs = mutableListOf() - val exitCode = if (isSimulator) { - runArgs.addAll(listOf( - "xcrun", "simctl", "launch", "--console", deviceId, bundleId, - "benchmarks=$benchmark", - "parallel=$parallel", - "warmupCount=100", - "modes=REAL", - "reportAtTheEnd=true" - )) - println(" $ ${runArgs.joinToString(" ")}") - execWithTee(runArgs, outFile) - } else { - runArgs.addAll(listOf( - "xcrun", "devicectl", "device", "process", "launch", "--console", "--device", deviceId, bundleId, - "--", - "benchmarks=$benchmark", - "parallel=$parallel", - "warmupCount=100", - "modes=REAL", - "reportAtTheEnd=true" - )) - println(" $ ${runArgs.joinToString(" ")}") - execWithTee(runArgs, outFile) - } +if (separateProcess) { + println(" ${benchmarksToRun.size} benchmarks individually\n") +} else { + println(" All benchmarks together\n") +} - if (exitCode != 0) { - System.out.format(" ⚠ WARNING: process exited with code %d\n", exitCode) - } else { - println(" ✓ done") - } +if (separateProcess) { + for ((index, benchmark) in benchmarksToRun.withIndex()) { + val outFileName = "${devicePrefix}_${benchmark}.txt" + val outFile = File(OUTPUT_DIR, outFileName) + + println(" [%3d/%3d] %-52s".format(index + 1, total, benchmark)) + println(" → ${outFile.name}") + + val finalAppArgs = appArgs.toMutableMap() + finalAppArgs["benchmarks"] = benchmark + + val exitCode = runBenchmark(deviceId, bundleId, isSimulator, finalAppArgs, outFile) - // Brief cooldown between runs so the device settles - println(" $ sleep 3") - Thread.sleep(3000) + if (exitCode != 0) { + println(" ⚠ WARNING: process exited with code $exitCode") + } else { + println(" ✓ done") } + + // Brief cooldown between runs so the device settles + println(" $ sleep 3") + Thread.sleep(3000) + } +} else { + val outFileName = "${devicePrefix}_all_benchmarks.txt" + val outFile = File(OUTPUT_DIR, outFileName) + + println(" [%3d/%3d] %-52s".format(1, total, "All Benchmarks")) + println(" → ${outFile.name}") + + val finalAppArgs = appArgs.toMutableMap() + finalAppArgs["benchmarks"] = benchmarksToRun.joinToString(",") + + val exitCode = runBenchmark(deviceId, bundleId, isSimulator, finalAppArgs, outFile) + + if (exitCode != 0) { + println(" ⚠ WARNING: process exited with code $exitCode") + } else { + println(" ✓ done") + } +} + +fun runBenchmark(deviceId: String, bundleId: String, isSimulator: Boolean, finalAppArgs: Map, outFile: File): Int { + val runArgs = mutableListOf() + val appArgsList = finalAppArgs.map { "${it.key}=${it.value}" } + + return if (isSimulator) { + runArgs.addAll(listOf("xcrun", "simctl", "launch", "--console", deviceId, bundleId)) + runArgs.addAll(appArgsList) + println(" $ ${runArgs.joinToString(" ")}") + execWithTee(runArgs, outFile) + } else { + runArgs.addAll(listOf("xcrun", "devicectl", "device", "process", "launch", "--console", "--device", deviceId, bundleId, "--")) + runArgs.addAll(appArgsList) + println(" $ ${runArgs.joinToString(" ")}") + execWithTee(runArgs, outFile) } } println("\n==> All done!") -System.out.format(" %d output files written to: %s\n\n", total, OUTPUT_DIR.path) +println(" %d output files written to: %s\n".format(total, OUTPUT_DIR.path)) From b2b00ce12d5e3d91c2287e75a2d4a86924bf0a96 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 19:14:53 +0200 Subject: [PATCH 03/10] exitProcess for non-real modes --- .../multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt index 48d0caaeee..373a4378ae 100644 --- a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt +++ b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt @@ -21,6 +21,7 @@ fun runBenchmarks() { MainScope().launch { runBenchmarks(graphicsContext = graphicsContext()) println("Completed!") + exitProcess(0) } } From 7913d134fe4d1f347185c9943226c047806bcd42 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 19:15:28 +0200 Subject: [PATCH 04/10] Align output directory --- .../multiplatform/run_ios_benchmarks.main.kts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/benchmarks/multiplatform/run_ios_benchmarks.main.kts b/benchmarks/multiplatform/run_ios_benchmarks.main.kts index 22f0e9893a..e01e9349de 100755 --- a/benchmarks/multiplatform/run_ios_benchmarks.main.kts +++ b/benchmarks/multiplatform/run_ios_benchmarks.main.kts @@ -12,7 +12,7 @@ import java.util.concurrent.TimeUnit * every benchmark (from Benchmarks.kt). * Console output is saved to: * - * benchmarks_result/__.txt + * build/benchmarks/text-reports/.txt * * Requirements: * - Xcode 15+ (uses xcrun devicectl for real devices, xcrun simctl for simulators) @@ -30,7 +30,8 @@ import java.util.concurrent.TimeUnit val ROOT_DIR = File(".").absoluteFile val PROJECT_DIR = File(ROOT_DIR, "iosApp") val MULTIPLATFORM_DIR = ROOT_DIR -val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "benchmarks_result") +val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "build/benchmarks") +val TEXT_REPORTS_DIR = File(OUTPUT_DIR, "text-reports") val SCHEME = "iosApp" val CONFIGURATION = "Release" val BUILD_DIR = File(MULTIPLATFORM_DIR, ".benchmark_build") @@ -218,7 +219,7 @@ println(" Name : $deviceName") println(" iOS : $deviceIos") println(" UDID : $deviceId") println(" Simulator : $isSimulator") -println(" Prefix : ${devicePrefix}_.txt") +println(" Filename : .txt") // ── 2. Build ─────────────────────────────────────────────────────────────────── @@ -313,11 +314,11 @@ if (separateProcess) { if (separateProcess) { for ((index, benchmark) in benchmarksToRun.withIndex()) { - val outFileName = "${devicePrefix}_${benchmark}.txt" - val outFile = File(OUTPUT_DIR, outFileName) + val (finalOutputDir, outFileName) = TEXT_REPORTS_DIR to "${benchmark}.txt" + val outFile = File(finalOutputDir, outFileName) println(" [%3d/%3d] %-52s".format(index + 1, total, benchmark)) - println(" → ${outFile.name}") + println(" → ${outFile.name} in ${finalOutputDir.relativeTo(ROOT_DIR).path}") val finalAppArgs = appArgs.toMutableMap() finalAppArgs["benchmarks"] = benchmark @@ -335,11 +336,11 @@ if (separateProcess) { Thread.sleep(3000) } } else { - val outFileName = "${devicePrefix}_all_benchmarks.txt" - val outFile = File(OUTPUT_DIR, outFileName) + val (finalOutputDir, outFileName) = TEXT_REPORTS_DIR to "all_benchmarks.txt" + val outFile = File(finalOutputDir, outFileName) println(" [%3d/%3d] %-52s".format(1, total, "All Benchmarks")) - println(" → ${outFile.name}") + println(" → ${outFile.name} in ${finalOutputDir.relativeTo(ROOT_DIR).path}") val finalAppArgs = appArgs.toMutableMap() finalAppArgs["benchmarks"] = benchmarksToRun.joinToString(",") @@ -371,4 +372,4 @@ fun runBenchmark(deviceId: String, bundleId: String, isSimulator: Boolean, final } println("\n==> All done!") -println(" %d output files written to: %s\n".format(total, OUTPUT_DIR.path)) +println(" %d output files written to: %s\n".format(total, TEXT_REPORTS_DIR.relativeTo(ROOT_DIR).path)) From 76d65830077325576f8f5f8997e45b3fbb33cd57 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 19:54:19 +0200 Subject: [PATCH 05/10] Support JSON saving for iOS target --- .../src/commonMain/kotlin/Benchmarks.kt | 10 ++++++ .../src/commonMain/kotlin/BenchmarksSave.kt | 2 ++ .../kotlin/BenchmarksSave.desktop.kt | 2 ++ .../benchmarks/src/iosMain/kotlin/main.ios.kt | 2 ++ .../src/macosMain/kotlin/main.macos.kt | 2 ++ .../src/webMain/kotlin/BenchmarksSave.web.kt | 2 ++ .../multiplatform/run_ios_benchmarks.main.kts | 35 ++++++++++++++++++- 7 files changed, 54 insertions(+), 1 deletion(-) diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt index ecdaebfef6..f72383de4f 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt @@ -327,6 +327,11 @@ suspend fun runBenchmark( content = benchmark.content ).generateStats() stats.prettyPrint() + if (Config.saveStatsToJSON && isIosTarget) { + println("JSON_START") + println(stats.toJsonString()) + println("JSON_END") + } if (Config.saveStats()) { saveBenchmarkStats(name = benchmark.name, stats = stats) } @@ -419,6 +424,11 @@ fun BenchmarkRunner( } else { results.add(stats) } + if (Config.saveStatsToJSON && isIosTarget) { + println("JSON_START") + println(stats.toJsonString()) + println("JSON_END") + } if (Config.saveStats()) { saveBenchmarkStats(name = benchmark.name, stats = stats) } diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt index a15fd76cd1..0eafa37834 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt @@ -70,6 +70,8 @@ fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) { */ expect fun saveBenchmarkStats(name: String, stats: BenchmarkStats) +expect val isIosTarget: Boolean + private fun RawSource.readText() = use { it.buffered().readByteArray().decodeToString() } diff --git a/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt b/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt index 815b87fb50..216db267c7 100644 --- a/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt +++ b/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt @@ -4,3 +4,5 @@ */ actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) = saveBenchmarkStatsOnDisk(name, stats) + +actual val isIosTarget: Boolean = false diff --git a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt index 373a4378ae..aa4327fadc 100644 --- a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt +++ b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt @@ -25,6 +25,8 @@ fun runBenchmarks() { } } +actual val isIosTarget: Boolean = true + @OptIn(ExperimentalComposeUiApi::class) fun MainViewController(): UIViewController { return ComposeUIViewController(configure = { parallelRendering = Config.parallelRendering }) { diff --git a/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt index 2c3f3dd0ce..9a1c8f0d03 100644 --- a/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt +++ b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt @@ -12,6 +12,8 @@ import platform.AppKit.NSScreen import platform.AppKit.maximumFramesPerSecond import kotlin.system.exitProcess +actual val isIosTarget: Boolean = true + fun main(args : Array) { Config.setGlobalFromArgs(args) if (Config.isModeEnabled(Mode.REAL)) { diff --git a/benchmarks/multiplatform/benchmarks/src/webMain/kotlin/BenchmarksSave.web.kt b/benchmarks/multiplatform/benchmarks/src/webMain/kotlin/BenchmarksSave.web.kt index e9854d7cac..e4d51a602a 100644 --- a/benchmarks/multiplatform/benchmarks/src/webMain/kotlin/BenchmarksSave.web.kt +++ b/benchmarks/multiplatform/benchmarks/src/webMain/kotlin/BenchmarksSave.web.kt @@ -30,6 +30,8 @@ actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) { } } +actual val isIosTarget: Boolean = false + /** * Client for sending benchmark results to the server */ diff --git a/benchmarks/multiplatform/run_ios_benchmarks.main.kts b/benchmarks/multiplatform/run_ios_benchmarks.main.kts index e01e9349de..2682620f8d 100755 --- a/benchmarks/multiplatform/run_ios_benchmarks.main.kts +++ b/benchmarks/multiplatform/run_ios_benchmarks.main.kts @@ -30,8 +30,9 @@ import java.util.concurrent.TimeUnit val ROOT_DIR = File(".").absoluteFile val PROJECT_DIR = File(ROOT_DIR, "iosApp") val MULTIPLATFORM_DIR = ROOT_DIR -val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "build/benchmarks") +val OUTPUT_DIR = File(MULTIPLATFORM_DIR, "benchmarks/build/benchmarks") val TEXT_REPORTS_DIR = File(OUTPUT_DIR, "text-reports") +val JSON_REPORTS_DIR = File(OUTPUT_DIR, "json-reports") val SCHEME = "iosApp" val CONFIGURATION = "Release" val BUILD_DIR = File(MULTIPLATFORM_DIR, ".benchmark_build") @@ -133,10 +134,23 @@ fun execWithTee(command: List, outputFile: File, workingDir: File = ROOT outputFile.parentFile.mkdirs() outputFile.bufferedWriter().use { writer -> var line: String? = reader.readLine() + var capturingJson = false + var capturedJson = StringBuilder() while (line != null) { println(line) writer.write(line) writer.newLine() + + if (line == "JSON_START") { + capturingJson = true + capturedJson = StringBuilder() + } else if (line == "JSON_END") { + capturingJson = false + saveCapturedJson(capturedJson.toString()) + } else if (capturingJson) { + capturedJson.append(line).append("\n") + } + line = reader.readLine() } } @@ -144,6 +158,25 @@ fun execWithTee(command: List, outputFile: File, workingDir: File = ROOT return process.waitFor() } +fun saveCapturedJson(json: String) { + try { + val jsonNode = json.trim() + if (jsonNode.isEmpty()) return + + // Extract benchmark name from JSON + val nameRegex = Regex("\"name\":\\s*\"([^\"]+)\"") + val match = nameRegex.find(jsonNode) + val benchmarkName = match?.groupValues?.get(1) ?: "unknown" + + JSON_REPORTS_DIR.mkdirs() + val jsonFile = File(JSON_REPORTS_DIR, "${benchmarkName}.json") + jsonFile.writeText(jsonNode) + println(" → Captured JSON saved to ${jsonFile.relativeTo(ROOT_DIR).path}") + } catch (e: Exception) { + println(" ⚠ Failed to save captured JSON: ${e.message}") + } +} + // ── 1. Detect target device or simulator ────────────────────────────────────── println("\n==> [1/4] Detecting target...") From a1c70fda274747baa25fd510eb8482a27ec7a7c3 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 20:19:42 +0200 Subject: [PATCH 06/10] Add iOS target to compare benchmarks script --- .../multiplatform/compare_benchmarks.main.kts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/benchmarks/multiplatform/compare_benchmarks.main.kts b/benchmarks/multiplatform/compare_benchmarks.main.kts index 5524d8614c..9f87b09992 100755 --- a/benchmarks/multiplatform/compare_benchmarks.main.kts +++ b/benchmarks/multiplatform/compare_benchmarks.main.kts @@ -17,7 +17,7 @@ fun main(args: Array) { val v2 = argMap["v2"] ?: args.getOrNull(1) if (v1 == null || v2 == null) { - println("Usage: compare_benchmarks.main.kts v1= v2= [runs=3] [benchmarks=] [platform=macos|desktop|web]") + println("Usage: ./compare_benchmarks.main.kts v1= v2= [runs=3] [benchmarks=] [platform=macos|desktop|web|ios]") return } @@ -26,28 +26,29 @@ fun main(args: Array) { val platform = argMap["platform"] ?: "macos" val runServer = platform == "web" val isWeb = platform == "web" + val isIos = platform == "ios" println("Comparing Compose versions: $v1 and $v2") println("Number of runs: $runs") println("Platform: $platform") benchmarkName?.let { println("Filtering by benchmark: $it") } - val resultsV1 = runBenchmarksForVersion(v1, runs, benchmarkName, platform, runServer, isWeb) - val resultsV2 = runBenchmarksForVersion(v2, runs, benchmarkName, platform, runServer, isWeb) + val resultsV1 = runBenchmarksForVersion(v1, runs, benchmarkName, platform, runServer, isWeb, isIos) + val resultsV2 = runBenchmarksForVersion(v2, runs, benchmarkName, platform, runServer, isWeb, isIos) compareResults(v1, resultsV1, v2, resultsV2) } data class BenchmarkResult(val name: String, val totalMs: Double) -fun runBenchmarksForVersion(version: String, runs: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean): Map> { +fun runBenchmarksForVersion(version: String, runs: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean, isIos: Boolean): Map> { println("\n=== Running benchmarks for version: $version ===") val allRunsResults = mutableMapOf>() for (i in 1..runs) { println("Run $i/$runs...") - executeBenchmarks(version, i, benchmarkName, platform, runServer, isWeb) + executeBenchmarks(version, i, benchmarkName, platform, runServer, isWeb, isIos) val runResults = collectResults(version, i) runResults.forEach { (name, value) -> allRunsResults.getOrPut(name) { mutableListOf() }.add(value) @@ -65,7 +66,7 @@ fun updateComposeVersion(version: String) { println("Updated gradle/libs.versions.toml to version $version") } -fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean) { +fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, platform: String, runServer: Boolean, isWeb: Boolean, isIos: Boolean) { val (versionedExecutable, defaultExecutable, task) = when (platform) { "macos" -> Triple( File("benchmarks/build/bin/macosArm64/releaseExecutable/benchmarks-$version.kexe"), @@ -82,6 +83,11 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl null, ":benchmarks:wasmJsBrowserProductionRun" ) + "ios" -> Triple( + null, + null, + "ios" + ) else -> throw IllegalArgumentException("Unsupported platform: $platform") } @@ -131,7 +137,7 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl Thread.sleep(5000) try { - executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, serverStopped) + executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, isIos, serverStopped) } finally { println("Stopping benchmark server...") serverProcess.destroy() @@ -140,7 +146,7 @@ fun executeBenchmarks(version: String, runIndex: Int, benchmarkName: String?, pl monitorThread.interrupt() } } else { - executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, null) + executeBenchmarksOnce(version, platform, task, runArgs, versionedExecutable, defaultExecutable, isWeb, isIos, null) } } @@ -152,8 +158,24 @@ fun executeBenchmarksOnce( versionedExecutable: File?, defaultExecutable: File?, isWeb: Boolean, + isIos: Boolean, serverStopped: java.util.concurrent.atomic.AtomicBoolean? ) { + if (isIos) { + println("Running version $version on iOS...") + updateComposeVersion(version) + val processBuilder = ProcessBuilder( + "./run_ios_benchmarks.main.kts", + *runArgs.toTypedArray() + ).inheritIO() + val process = processBuilder.start() + val exitCode = process.waitFor() + if (exitCode != 0) { + println("Warning: iOS Benchmark run failed with exit code $exitCode") + } + return + } + if (versionedExecutable != null && versionedExecutable.exists()) { println("Using existing executable for version $version: ${versionedExecutable.absolutePath}") From f99f52fc745c175d904907f2077f13f8c6a94449 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Mon, 23 Mar 2026 20:53:27 +0200 Subject: [PATCH 07/10] Fetch available benchmarks list in the run_ios_benchmarks scripts --- .../src/commonMain/kotlin/Benchmarks.kt | 6 +++ .../src/commonMain/kotlin/Config.kt | 15 ++++++-- .../iosApp/run_ios_benchmarks.sh | 27 ++++++-------- .../multiplatform/run_ios_benchmarks.main.kts | 37 +++++++++++-------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt index f72383de4f..f36dd805f7 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt @@ -346,6 +346,12 @@ suspend fun runBenchmarks( warmupCount: Int = Config.warmupCount, graphicsContext: GraphicsContext? = null ) { + if (Config.listBenchmarks) { + println("AVAILABLE_BENCHMARKS_START") + benchmarks.forEach { println(it.name) } + println("AVAILABLE_BENCHMARKS_END") + return + } println() println("Running emulating $targetFps FPS") println() diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt index c0ccac9bcd..0c37350b8b 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt @@ -27,7 +27,7 @@ object Args { * @param args an array of strings representing the command line arguments. * Each argument can specify either of these settings: * modes, benchmarks, disabledBenchmarks - comma separated values, - * versionInfo, saveStatsToCSV, saveStatsToJSON, parallel, warmupCount, frameCount, emptyScreenDelay, reportAtTheEnd - single values. + * versionInfo, saveStatsToCSV, saveStatsToJSON, parallel, warmupCount, frameCount, emptyScreenDelay, reportAtTheEnd, listBenchmarks - single values. * * Example: benchmarks=AnimatedVisibility(100),modes=SIMPLE,versionInfo=Kotlin_2_1_20,saveStatsToCSV=true,warmupCount=50,frameCount=100,emptyScreenDelay=2000,reportAtTheEnd=true */ @@ -44,6 +44,7 @@ object Args { var frameCount: Int? = null var emptyScreenDelay: Long? = null var reportAtTheEnd: Boolean = false + var listBenchmarks: Boolean = false for (arg in args) { if (arg.startsWith("modes=", ignoreCase = true)) { @@ -70,6 +71,8 @@ object Args { emptyScreenDelay = arg.substringAfter("=").toLong() } else if (arg.startsWith("reportAtTheEnd=", ignoreCase = true)) { reportAtTheEnd = arg.substringAfter("=").toBoolean() + } else if (arg.startsWith("listBenchmarks=", ignoreCase = true)) { + listBenchmarks = arg.substringAfter("=").toBoolean() } else { println("WARNING: unknown argument $arg") } @@ -89,7 +92,8 @@ object Args { warmupCount = warmupCount ?: defaultWarmupCount, frameCount = frameCount ?: 1000, emptyScreenDelay = emptyScreenDelay ?: 2000L, - reportAtTheEnd = reportAtTheEnd + reportAtTheEnd = reportAtTheEnd, + listBenchmarks = listBenchmarks ) } } @@ -110,6 +114,7 @@ object Args { * @property frameCount Number of frames to run for each benchmark. * @property emptyScreenDelay Delay in milliseconds between warmup and benchmark. * @property reportAtTheEnd Flag indicating whether we should report results at the end of all benchmarks. + * @property listBenchmarks Flag indicating whether we should print available benchmarks and exit. */ data class Config( val modes: Set = emptySet(), @@ -123,7 +128,8 @@ data class Config( val warmupCount: Int = 100, val frameCount: Int = 1000, val emptyScreenDelay: Long = 2000L, - val reportAtTheEnd: Boolean = false + val reportAtTheEnd: Boolean = false, + val listBenchmarks: Boolean = false ) { /** * Checks if a specific mode is enabled based on the configuration. @@ -185,6 +191,9 @@ data class Config( val reportAtTheEnd: Boolean get() = global.reportAtTheEnd + val listBenchmarks: Boolean + get() = global.listBenchmarks + fun setGlobal(global: Config) { this.global = global } diff --git a/benchmarks/multiplatform/iosApp/run_ios_benchmarks.sh b/benchmarks/multiplatform/iosApp/run_ios_benchmarks.sh index 77d4065d44..e15d9e1ba1 100755 --- a/benchmarks/multiplatform/iosApp/run_ios_benchmarks.sh +++ b/benchmarks/multiplatform/iosApp/run_ios_benchmarks.sh @@ -32,21 +32,6 @@ CONFIGURATION="Release" ATTEMPTS=5 BUILD_DIR="$MULTIPLATFORM_DIR/.benchmark_build" -BENCHMARKS=( - "AnimatedVisibility" - "LazyGrid" - "LazyGrid-ItemLaunchedEffect" - "LazyGrid-SmoothScroll" - "LazyGrid-SmoothScroll-ItemLaunchedEffect" - "VisualEffects" - "LazyList" - "MultipleComponents" - "MultipleComponents-NoVectorGraphics" - "TextLayout" - "CanvasDrawing" - "HeavyShader" -) - # ── Helpers ──────────────────────────────────────────────────────────────────── die() { echo ""; echo "ERROR: $*" >&2; exit 1; } @@ -192,6 +177,18 @@ mkdir -p "$OUTPUT_DIR" # ── 4. Run benchmarks ────────────────────────────────────────────────────────── +echo "" +echo "==> [4/4] Fetching benchmarks list via Gradle..." +TEMP_LIST=$(mktemp) +(cd "$MULTIPLATFORM_DIR" && ./gradlew :benchmarks:run -PrunArguments=listBenchmarks=true) > "$TEMP_LIST" 2>&1 + +BENCHMARKS=($(awk '/AVAILABLE_BENCHMARKS_START/{p=1;next} /AVAILABLE_BENCHMARKS_END/{p=0} p && NF{print}' "$TEMP_LIST")) +rm "$TEMP_LIST" + +if [[ ${#BENCHMARKS[@]} -eq 0 ]]; then + die "No benchmarks found to run." +fi + TOTAL=$(( ${#BENCHMARKS[@]} * 2 * ATTEMPTS )) CURRENT=0 diff --git a/benchmarks/multiplatform/run_ios_benchmarks.main.kts b/benchmarks/multiplatform/run_ios_benchmarks.main.kts index 2682620f8d..6ddff4000b 100755 --- a/benchmarks/multiplatform/run_ios_benchmarks.main.kts +++ b/benchmarks/multiplatform/run_ios_benchmarks.main.kts @@ -37,21 +37,6 @@ val SCHEME = "iosApp" val CONFIGURATION = "Release" val BUILD_DIR = File(MULTIPLATFORM_DIR, ".benchmark_build") -val DEFAULT_BENCHMARKS = listOf( - "AnimatedVisibility", - "LazyGrid", - "LazyGrid-ItemLaunchedEffect", - "LazyGrid-SmoothScroll", - "LazyGrid-SmoothScroll-ItemLaunchedEffect", - "VisualEffects", - "LazyList", - "MultipleComponents", - "MultipleComponents-NoVectorGraphics", - "TextLayout", - "CanvasDrawing", - "HeavyShader" -) - // ── Argument Parsing ────────────────────────────────────────────────────────── val parsedArgs = mutableMapOf() @@ -78,7 +63,6 @@ rawArgs.forEach { arg -> } val benchmarksFromArgs = parsedArgs["benchmarks"]?.split(",")?.map { it.substringBefore("(").trim() }?.filter { it.isNotEmpty() } -val benchmarksToRun = benchmarksFromArgs ?: DEFAULT_BENCHMARKS val separateProcess = parsedArgs["separateprocess"]?.toBoolean() ?: true // Arguments to pass to the app @@ -336,6 +320,27 @@ OUTPUT_DIR.mkdirs() // ── 4. Run benchmarks ────────────────────────────────────────────────────────── +fun fetchBenchmarks(): List { + try { + val output = exec("./gradlew", ":benchmarks:run", "-PrunArguments=listBenchmarks=true") + val lines = output.lines() + val startIndex = lines.indexOf("AVAILABLE_BENCHMARKS_START") + val endIndex = lines.indexOf("AVAILABLE_BENCHMARKS_END") + if (startIndex != -1 && endIndex != -1 && endIndex > startIndex) { + return lines.subList(startIndex + 1, endIndex).filter { it.isNotBlank() } + } + } catch (e: Exception) { + println(" ⚠ Failed to fetch benchmarks list via Gradle: ${e.message}") + } + return emptyList() +} + +val benchmarksToRun = if (benchmarksFromArgs == null || benchmarksFromArgs.isEmpty()) fetchBenchmarks() else benchmarksFromArgs + +if (benchmarksToRun.isEmpty()) { + die("No benchmarks found to run.") +} + val total = if (separateProcess) benchmarksToRun.size else 1 println("\n==> [4/4] Running $total benchmark sessions") From acb68eb1fa42d72fcca15849a46a753742bacca3 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Tue, 24 Mar 2026 13:24:23 +0200 Subject: [PATCH 08/10] Correct README.md --- benchmarks/multiplatform/README.md | 60 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/benchmarks/multiplatform/README.md b/benchmarks/multiplatform/README.md index 5402234ebf..726019fc0f 100644 --- a/benchmarks/multiplatform/README.md +++ b/benchmarks/multiplatform/README.md @@ -1,5 +1,48 @@ # Compose Multiplatform benchmarks +This project contains performance benchmarks for Compose Multiplatform on various targets, +including Desktop, iOS, MacOS, and Web (Kotlin/Wasm and Kotlin/JS). +These benchmarks measure the performance of various Compose components and features, +such as animations, lazy layouts, text rendering, and visual effects. + +## Benchmark Modes + +The benchmarks can be run in different modes, which determine how performance is measured and reported: + +- **`SIMPLE`**: Measures basic frame times without considering VSync. Good for quick checks of raw rendering performance. Enabled by default if no modes are specified. +- **`VSYNC_EMULATION`**: Emulates VSync behavior to estimate missed frames and provide more realistic performance metrics (CPU/GPU percentiles). Enabled by default if no modes are specified. +- **`REAL`**: Runs the benchmark in a real-world scenario with actual VSync. This mode provides the most accurate results for user-perceived performance (FPS, actual missed frames) + but may not catch performance regressions if a frame fits to budget. + Also requires a device with a real display (may have problems with headless devices). + +To enable specific modes, use the `modes` argument: +`modes=SIMPLE,VSYNC_EMULATION,REAL` + +## Configuration Arguments + +You can configure benchmark runs using arguments passed to the Gradle task (via `-PrunArguments="..."`) or to the `.main.kts` script. Arguments can also be set in `gradle.properties` using the `runArguments` property. + +| Argument | Description | Example | +|----------|------------------------------------------------------------|---------| +| `modes` | Comma-separated list of execution modes (`SIMPLE`, `VSYNC_EMULATION`, `REAL`). | `modes=REAL` | +| `benchmarks` | Comma-separated list of benchmarks to run. Can optionally specify problem size in parentheses. | `benchmarks=LazyGrid(100),AnimatedVisibility` | +| `disabledBenchmarks` | Comma-separated list of benchmarks to skip. | `disabledBenchmarks=HeavyShader` | +| `warmupCount` | Number of warmup frames before starting measurements. | `warmupCount=50` | +| `frameCount` | Number of frames to measure for each benchmark. | `frameCount=500` | +| `emptyScreenDelay` | Delay in milliseconds between warmup and measurement (real mode only).| `emptyScreenDelay=1000` | +| `parallel` | (iOS only) Enable parallel rendering. | `parallel=true` | +| `saveStatsToCSV` | Save results to CSV files. | `saveStatsToCSV=true` | +| `saveStatsToJSON` | Save results to JSON files. | `saveStatsToJSON=true` | +| `versionInfo` | Add version information to the report. | `versionInfo=1.2.3` | +| `reportAtTheEnd` | Print a summary report after all benchmarks are finished real mode only).| `reportAtTheEnd=true` | +| `listBenchmarks` | List all available benchmarks and exit. | `listBenchmarks=true` | + +### Usage Example + +```bash +./gradlew :benchmarks:run -PrunArguments="benchmarks=LazyGrid modes=REAL frameCount=200" +``` + ## Run Desktop - `./gradlew :benchmarks:run` @@ -8,15 +51,24 @@ Open the project in Fleet or Android Studio with KMM plugin installed and choose `iosApp` run configuration. Make sure that you build the app in `Release` configuration. Alternatively you may open `iosApp/iosApp` project in XCode and run the app from there. -## Run automated iOS benchmarks +## Run iOS benchmarks via scripts 1. To run on device, open `iosApp/iosApp.xcodeproj` and properly configure the Signing section on the Signing & Capabilities project tab. 2. Use the following command to get list of all iOS devices: - `xcrun xctrace list devices` 3. From the benchmarks directory run: -- `./iosApp/run_ios_benchmarks.sh ` -4. Results are saved as `.txt` files in `benchmarks_result/`. +- `./run_ios_benchmarks.main.kts ` (supports all modes of running benchmarks, configured the same way as for other targets: + script arguments or `runArguments` property of `gradle.properties`) +- or `./iosApp/run_ios_benchmarks.sh ` (shell script supporting `real` mode benchmarks running with multiple attempts) + +To run specific benchmarks: +- `./run_ios_benchmarks.main.kts benchmarks=AnimatedVisibility,LazyGrid` + +To run all benchmarks in a single process (faster, but may be less stable): +- `./run_ios_benchmarks.main.kts separateProcess=false` + +4. Results are saved in `benchmarks/build/benchmarks/text-reports/` (when using `.main.kts`) or `benchmarks_result/` (when using `.sh`). -## Run native on MacOS + ## Run native on MacOS - `./gradlew :benchmarks:runReleaseExecutableMacosArm64` (Works on Arm64 processors) - `./gradlew :benchmarks:runReleaseExecutableMacosX64` (Works on Intel processors) From 7cb536ec7bc5abeb43d2b7f535be3d9768e225e8 Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Thu, 26 Mar 2026 15:23:03 +0200 Subject: [PATCH 09/10] Review fixes in README.md --- benchmarks/multiplatform/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/multiplatform/README.md b/benchmarks/multiplatform/README.md index 726019fc0f..c57e01b4bd 100644 --- a/benchmarks/multiplatform/README.md +++ b/benchmarks/multiplatform/README.md @@ -63,12 +63,13 @@ Alternatively you may open `iosApp/iosApp` project in XCode and run the app from To run specific benchmarks: - `./run_ios_benchmarks.main.kts benchmarks=AnimatedVisibility,LazyGrid` -To run all benchmarks in a single process (faster, but may be less stable): +To run all benchmarks in a single process (faster but may be less reliable as some +benchmarks may affect others within one process): - `./run_ios_benchmarks.main.kts separateProcess=false` 4. Results are saved in `benchmarks/build/benchmarks/text-reports/` (when using `.main.kts`) or `benchmarks_result/` (when using `.sh`). - ## Run native on MacOS +## Run native on MacOS - `./gradlew :benchmarks:runReleaseExecutableMacosArm64` (Works on Arm64 processors) - `./gradlew :benchmarks:runReleaseExecutableMacosX64` (Works on Intel processors) From 32935769e0d461ce987272997b6199f2857c1c0e Mon Sep 17 00:00:00 2001 From: Nikita Lipsky Date: Thu, 26 Mar 2026 15:38:34 +0200 Subject: [PATCH 10/10] Prevent screen from dimming on iOS for non-real modes. --- .../multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt index aa4327fadc..e19a0250e6 100644 --- a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt +++ b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt @@ -7,6 +7,7 @@ import androidx.compose.ui.ExperimentalComposeUiApi import androidx.compose.ui.window.ComposeUIViewController import kotlinx.coroutines.MainScope import kotlinx.coroutines.launch +import platform.UIKit.UIApplication import platform.UIKit.UIScreen import platform.UIKit.UIViewController import kotlin.system.exitProcess @@ -18,6 +19,7 @@ fun setGlobalFromArgs(args : List) { fun runReal() = Config.isModeEnabled(Mode.REAL) fun runBenchmarks() { + UIApplication.sharedApplication.setIdleTimerDisabled(true) MainScope().launch { runBenchmarks(graphicsContext = graphicsContext()) println("Completed!")