Skip to content

test: balance unit test shards by isolating heavy test (and fix shard index bug) #563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fglock merged 1 commit into from
Apr 26, 2026
Merged

test: balance unit test shards by isolating heavy test (and fix shard index bug) #563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ check-java-gradle:

wrapper: check-java-gradle

# Standard build - incremental compilation with parallel tests (4 JVMs)
# Standard build - incremental compilation with parallel tests (5 JVMs; last shard isolates heavy tests)
build: check-java-gradle
ifeq ($(OS),Windows_NT)
gradlew.bat classes testUnitParallel --parallel shadowJar
Expand Down Expand Up @@ -149,20 +149,20 @@ else
./gradlew testAll --rerun-tasks
endif

# Parallel unit tests via Gradle/JUnit (4 JVMs)
# Parallel unit tests via Gradle/JUnit (5 JVMs; last shard isolates heavy tests)
test-gradle-parallel: check-java-gradle
ifeq ($(OS),Windows_NT)
gradlew.bat testUnitParallel --parallel --rerun-tasks
else
./gradlew testUnitParallel --parallel --rerun-tasks
endif

# Parallel unit tests via Maven (4 JVMs)
# Parallel unit tests via Maven (5 JVMs; last shard isolates heavy tests)
test-maven-parallel:
ifeq ($(OS),Windows_NT)
start /B mvn test -Pshard1 & start /B mvn test -Pshard2 & start /B mvn test -Pshard3 & start /B mvn test -Pshard4
start /B mvn test -Pshard1 & start /B mvn test -Pshard2 & start /B mvn test -Pshard3 & start /B mvn test -Pshard4 & start /B mvn test -Pshard5
else
mvn test -Pshard1 & mvn test -Pshard2 & mvn test -Pshard3 & mvn test -Pshard4 & wait
mvn test -Pshard1 & mvn test -Pshard2 & mvn test -Pshard3 & mvn test -Pshard4 & mvn test -Pshard5 & wait
endif

clean: check-java-gradle
Expand Down
10 changes: 8 additions & 2 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,12 @@ tasks.named('processTestResources', Copy) {

// Parallel test execution tasks
// Run with: ./gradlew testUnitParallel --parallel
def parallelShards = 4
//
// Shard layout: the LAST shard is dedicated to a small set of known-heavy
// tests (see HEAVY_TESTS in PerlScriptExecutionTest.java). The other shards
// round-robin the remaining tests. This keeps wall-time roughly balanced
// even when one test dominates (e.g. unit/code_too_large.t).
def parallelShards = 5

(0..<parallelShards).each { index ->
tasks.register("testUnitShard${index}", Test) {
Expand All @@ -425,7 +430,8 @@ def parallelShards = 4
tasks.register('testUnitParallel') {
group = 'verification'
description = 'Runs unit tests in parallel across multiple JVMs. Usage: gradle testUnitParallel --parallel'
dependsOn 'testUnitShard0', 'testUnitShard1', 'testUnitShard2', 'testUnitShard3'
def shardTasks = (0..<parallelShards).collect { "testUnitShard${it}" }
dependsOn shardTasks
}

// Version catalog update configuration
Expand Down
29 changes: 23 additions & 6 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,11 @@
</build>

<!-- Profiles for parallel test execution -->
<!-- Usage: Run all 4 shards in parallel with separate terminal windows or background processes:
mvn test -Pshard1 & mvn test -Pshard2 & mvn test -Pshard3 & mvn test -Pshard4 & wait
<!-- Usage: Run all 5 shards in parallel with separate terminal windows or background processes:
mvn test -Pshard1 & mvn test -Pshard2 & mvn test -Pshard3 & mvn test -Pshard4 & mvn test -Pshard5 & wait

The last shard (shard5) is dedicated to known-heavy tests; see
HEAVY_TESTS in PerlScriptExecutionTest.java.
-->
<profiles>
<profile>
Expand All @@ -233,7 +236,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=1 -Dtest.shard.total=4</argLine>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=0 -Dtest.shard.total=5</argLine>
</configuration>
</plugin>
</plugins>
Expand All @@ -247,7 +250,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=2 -Dtest.shard.total=4</argLine>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=1 -Dtest.shard.total=5</argLine>
</configuration>
</plugin>
</plugins>
Expand All @@ -261,7 +264,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=3 -Dtest.shard.total=4</argLine>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=2 -Dtest.shard.total=5</argLine>
</configuration>
</plugin>
</plugins>
Expand All @@ -275,7 +278,21 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=4 -Dtest.shard.total=4</argLine>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=3 -Dtest.shard.total=5</argLine>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>shard5</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-native-access=ALL-UNNAMED -Dtest.shard.index=4 -Dtest.shard.total=5</argLine>
</configuration>
</plugin>
</plugins>
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/perlonjava/core/Configuration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,22 @@ public final class Configuration {
* Automatically populated by Gradle/Maven during build.
* DO NOT EDIT MANUALLY - this value is replaced at build time.
*/
public static final String gitCommitId = "0663a8089";
public static final String gitCommitId = "2c7f37913";

/**
* Git commit date of the build (ISO format: YYYY-MM-DD).
* Automatically populated by Gradle/Maven during build.
* DO NOT EDIT MANUALLY - this value is replaced at build time.
*/
public static final String gitCommitDate = "2026-04-25";
public static final String gitCommitDate = "2026-04-26";

/**
* Build timestamp in Perl 5 "Compiled at" format (e.g., "Apr 7 2026 11:20:00").
* Automatically populated by Gradle during build.
* Parsed by App::perlbrew and other tools via: perl -V | grep "Compiled at"
* DO NOT EDIT MANUALLY - this value is replaced at build time.
*/
public static final String buildTimestamp = "Apr 26 2026 09:17:16";
public static final String buildTimestamp = "Apr 26 2026 17:32:55";

// Prevent instantiation
private Configuration() {
Expand Down
56 changes: 44 additions & 12 deletions src/test/java/org/perlonjava/PerlScriptExecutionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,27 +122,59 @@ private static Stream<String> getPerlScripts(boolean unitOnly) throws IOExceptio
return sortedScripts.stream();
}

// Sharding logic
// Sharding logic.
//
// We use a simple "heavy-test isolation" scheme for balance:
// - The LAST shard is dedicated to known-heavy tests (HEAVY_TESTS).
// - All other shards round-robin the remaining (light) tests.
//
// Rationale: a single test (currently unit/code_too_large.t at ~16s)
// can dominate one shard's wall time well beyond what round-robin
// can balance. Putting it on its own shard lets the others split
// the rest evenly. To extend, just add filenames to HEAVY_TESTS.
String shardIndexProp = System.getProperty("test.shard.index");
String shardTotalProp = System.getProperty("test.shard.total");
if (shardIndexProp != null && !shardIndexProp.isEmpty() &&

if (shardIndexProp != null && !shardIndexProp.isEmpty() &&
shardTotalProp != null && !shardTotalProp.isEmpty()) {
try {
int shardIndex = Integer.parseInt(shardIndexProp);
int shardTotal = Integer.parseInt(shardTotalProp);

// Maven surefire.forkNumber is 1-indexed, convert to 0-indexed
if (shardIndex >= 1 && shardIndex <= shardTotal) {
shardIndex = shardIndex - 1;
}


// Both Gradle and Maven now pass 0-indexed shard.index values
// (see build.gradle and pom.xml). Earlier code attempted to
// detect Maven 1-indexed values heuristically, which silently
// collapsed shard 1 onto shard 0 and dropped the last shard
// entirely; do not reintroduce that.

if (shardTotal > 1 && shardIndex >= 0 && shardIndex < shardTotal) {
System.out.println("Running shard " + (shardIndex + 1) + " of " + shardTotal);

// Tests known to be much slower than the rest. Use forward
// slashes; we match against both '/' and '\' separators.
final List<String> HEAVY_TESTS = List.of(
"unit/code_too_large.t"
);
java.util.function.Predicate<String> isHeavy = s -> {
String norm = s.replace('\\', '/');
return HEAVY_TESTS.contains(norm);
};

final int lastShard = shardTotal - 1;
if (shardIndex == lastShard) {
// Dedicated shard: only the heavy tests.
return sortedScripts.stream().filter(isHeavy);
}

// Light shards: round-robin over the remaining (shardTotal - 1) shards.
List<String> lightScripts = sortedScripts.stream()
.filter(isHeavy.negate())
.collect(Collectors.toList());
final int lightShards = shardTotal - 1;
final int finalShardIndex = shardIndex;
return IntStream.range(0, sortedScripts.size())
.filter(i -> i % shardTotal == finalShardIndex)
.mapToObj(sortedScripts::get);
return IntStream.range(0, lightScripts.size())
.filter(i -> i % lightShards == finalShardIndex)
.mapToObj(lightScripts::get);
}
} catch (NumberFormatException e) {
// Silently fall through to run all tests
Expand Down
Loading