try make tests run and pass

.github/workflows/ci.yml

@@ -11,12 +11,89 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    strategy:
+      fail-fast: false
+      matrix:
+        module:
+          - useCases/fabiland 
+          - useCases/austin
+          - useCases/bangkok
+          - useCases/berlinBrandenburg
+          - useCases/capeTown
+          - useCases/kagawa
+          - useCases/manchester
+          - useCases/maryland
+          - useCases/munich
+          - useCases/perth
+          - siloCore
+          - analysis
+          - extensions
+          - synthetic-population
+
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin' # See 'Supported distributions' for available options
           java-version: '21'
-      - name: Build with Maven
-        run: mvn test --fail-at-end --batch-mode -Dmatsim.preferLocalDtds=true
+          cache: 'maven'
+
+      - name: Install HDF5 native dependencies (for hdf-java)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libhdf5-dev
+
+      # - name: Build with Maven
+      #   run: mvn test --fail-at-end --batch-mode -Dmatsim.preferLocalDtds=true
+# the orriginal command
+
+      - name: Build module (with dependencies)
+        run: mvn install --batch-mode --also-make --projects ${{matrix.module}} -DskipTests -Dsource.skip
+
+      - name: Test module
+        run: mvn verify --batch-mode -Dmaven.test.redirectTestOutputToFile -Dmatsim.preferLocalDtds=true --fail-at-end -Dsource.skip
+        working-directory: ${{matrix.module}}
+
+      - name: Sanitize module name for artifact
+        if: failure()
+        id: sanitize
+        run: echo "name=$(echo '${{ matrix.module }}' | tr '/' '-')" >> $GITHUB_OUTPUT
+
+      - name: Upload test output on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: test-output-${{ steps.sanitize.outputs.name }}
+          path: |
+            ${{ matrix.module }}/test/output/
+            ${{ matrix.module }}/target/surefire-reports/
+          retention-days: 5
+
+    env:
+      MAVEN_OPTS: -Xmx2g
+
+# the following is what matsim-libs has.  Don't know why it is needed.  kai, apr'26
+
+  verify-all-jobs-successful:
+    # always() - to ensure this job is executed (regardless of the status of the previous job)
+    # run if push or pull_requests from fork
+    if: always() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository)
+    needs: build
+    runs-on: ubuntu-latest
+
+    # When running this workflow for internal PRs "verify-all-jobs-successful" is marked as skipped,
+    # which GitHub takes as a green flag and will allow merging before "verify-all-jobs-successful"
+    # is completed for the push event.
+    # One way to avoid this is to create a job from a matrix on the fly. They will be created for
+    # push events and PR events from forks (but not for PR events coming from internal forks),
+    # so we can safely add a check "verify-all-jobs-successful (push-or-external-PR)"
+    strategy:
+      matrix:
+        name: [ "push-or-external-PR" ]
+
+    steps:
+      - name: check if the whole job matrix is successful
+        if: needs.build.result != 'success'
+        run: exit 1 # fail if "build" was not successful

README.md

@@ -24,3 +24,4 @@ A possibility is
 
 -----
 
+

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimTransportModel.java

@@ -43,11 +43,14 @@
 import org.matsim.vehicles.Vehicle;
 import org.matsim.vehicles.VehicleType;
 import org.matsim.vehicles.VehicleUtils;
+import org.matsim.vehicles.VehiclesFactory;
 
 import java.io.File;
 
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 
 import static de.tum.bgu.msm.properties.modules.TransportModelPropertiesModule.*;
 
@@ -91,6 +94,8 @@ public MatsimTransportModel(DataContainer dataContainer, Config matsimConfig,
 
     @Override
     public void setup() {
+        logger.warn("### Just entered MatsimTransportModel#setup().  This read from events if available, otherwise run the transport model.");
+
         internalTravelTimes.initialize(dataContainer.getGeoData(), matsimData);
 
         if (properties.transportModel.matsimInitialEventsFile == null) {
@@ -99,6 +104,7 @@ public void setup() {
             String eventsFile = properties.main.baseDirectory + properties.transportModel.matsimInitialEventsFile;
             replayFromEvents(eventsFile);
         }
+        logger.warn("### About to leave MatsimTransportModel#setup()." );
     }
 
     @Override
@@ -118,7 +124,10 @@ public void endSimulation() {
 
     private void runTransportModel(int year) {
         logger.warn("Running MATSim transport model for year " + year + ".");
-        Scenario assembledScenario = ScenarioUtils.createScenario(initialMatsimConfig);
+
+        Scenario assembledScenario = matsimData.getScenario();
+		PopulationFactory pf = assembledScenario.getPopulation().getFactory();
+
         TravelTimes travelTimes = dataContainer.getTravelTimes();
 
         if (year == properties.main.baseYear && properties.transportModel.transportModelIdentifier == TransportModelIdentifier.MATSIM){
@@ -127,29 +136,49 @@ private void runTransportModel(int year) {
             TravelDisutility myTravelDisutility = SiloMatsimUtils.getAnEmptyNetworkTravelDisutility();
             updateTravelTimes(myTravelTime, myTravelDisutility);
         }
+		{
+			Set<Id<org.matsim.api.core.v01.population.Person>> toRemove = new HashSet<>( assembledScenario.getPopulation().getPersons().keySet() );
+			for( Id<org.matsim.api.core.v01.population.Person> personId : toRemove ){
+				assembledScenario.getPopulation().removePerson( personId );
+			}
+		}
+		{
+			Set<Id<Vehicle>> toRemove = new HashSet<>( assembledScenario.getVehicles().getVehicles().keySet() );
+			for( Id<Vehicle> personId : toRemove ){
+				assembledScenario.getVehicles().removeVehicle( personId );
+			}
+		}
 
         for (Household household: dataContainer.getHouseholdDataManager().getHouseholds()) {
             for (Person pp : household.getPersons().values()) {
-                PopulationFactory pf = assembledScenario.getPopulation().getFactory();
-
                 final int noHHAUtos = (int) household.getVehicles().stream().filter( vv -> vv.getType().equals( de.tum.bgu.msm.data.vehicle.VehicleType.CAR ) ).count();
                 assembledScenario.getPopulation().addPerson( SiloMatsimUtils.createMatsimAlterEgo(pf, pp, noHHAUtos ) );
             }
         }
 
-        matsimData.updateMatsimPopulation(assembledScenario.getPopulation());
+//        matsimData.updateMatsimPopulation(assembledScenario.getPopulation());
+//		matsimData.updateVehicles( assembledScenario.getVehicles() );
 
         // create a dummy vehicle type
-        VehicleType dummyVehType = assembledScenario.getVehicles().getFactory().createVehicleType(Id.create("defaultVehicleType", VehicleType.class));
-        assembledScenario.getVehicles().addVehicleType(dummyVehType);
+		final VehiclesFactory vf = assembledScenario.getVehicles().getFactory();
+
+		final Id<VehicleType> defaultVehicleTypeId = Id.create( "defaultVehicleType", VehicleType.class );
+		VehicleType dummyVehType = assembledScenario.getVehicles().getVehicleTypes().get( defaultVehicleTypeId );
+		if ( dummyVehType==null ){
+			dummyVehType = vf.createVehicleType( defaultVehicleTypeId );
+			assembledScenario.getVehicles().addVehicleType( dummyVehType );
+		}
 
         for (org.matsim.api.core.v01.population.Person person : assembledScenario.getPopulation().getPersons().values()) {
             Id<Vehicle> vehicleId = Id.createVehicleId(person.getId());
-            assembledScenario.getVehicles().addVehicle(assembledScenario.getVehicles().getFactory().createVehicle(vehicleId, dummyVehType));
+            assembledScenario.getVehicles().addVehicle( vf.createVehicle(vehicleId, dummyVehType ) );
             VehicleUtils.insertVehicleIdsIntoAttributes(person, Collections.singletonMap( TransportMode.car, vehicleId) );
         }
 
+		// === I cannot say how the above and the below "assembledScenario" are connected.  Or not.  !!!!!!!
+
         // TODO remove config argument as it is duplicate (cf. above)
+		// yyyyyy the following replaces "assembledScenario"!!!
         assembledScenario = scenarioAssembler.assembleScenario(initialMatsimConfig, year, travelTimes);
 
         finalizeConfig(assembledScenario.getConfig(), year, Integer.toString(year));
@@ -158,8 +187,9 @@ private void runTransportModel(int year) {
 
         controler.run();
         logger.warn("Running MATSim transport model for year " + year + " finished.");
-
+        logger.warn("###################################################");
         // Get travel Times from MATSim
+        logger.warn("###################################################");
         logger.warn("Using MATSim to compute travel times from zone to zone.");
         TravelTime travelTime = controler.getLinkTravelTimes();
         TravelDisutility travelDisutility = controler.getTravelDisutilityFactory().createTravelDisutility(travelTime);
@@ -173,7 +203,10 @@ private void finalizeConfig(Config config, int runId, String dir) {
         config.controller().setOutputDirectory(outputDirectory);
         config.controller().setWritePlansInterval(Math.max(config.controller().getLastIteration(), 1));
         config.controller().setWriteEventsInterval(Math.max(config.controller().getLastIteration(), 1));
-        config.controller().setOverwriteFileSetting(OutputDirectoryHierarchy.OverwriteFileSetting.deleteDirectoryIfExists);
+
+//        config.controller().setOverwriteFileSetting(OutputDirectoryHierarchy.OverwriteFileSetting.deleteDirectoryIfExists);
+		// --> otherwise, it overwrites the result from the matsim main run during the accessibility computation
+
         config.routing().setRoutingRandomness(0.);
     }
 

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimTravelTimesAndCosts.java

@@ -14,6 +14,7 @@
 import org.apache.logging.log4j.Logger;
 import org.matsim.api.core.v01.Coord;
 import org.matsim.api.core.v01.Id;
+import org.matsim.api.core.v01.Scenario;
 import org.matsim.api.core.v01.TransportMode;
 import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.network.Network;
@@ -32,6 +33,10 @@
 import org.matsim.facilities.ActivityFacilitiesFactoryImpl;
 import org.matsim.facilities.ActivityFacility;
 import org.matsim.facilities.Facility;
+import org.matsim.vehicles.Vehicle;
+import org.matsim.vehicles.VehicleType;
+import org.matsim.vehicles.VehicleUtils;
+import org.matsim.vehicles.VehiclesFactory;
 
 import java.util.*;
 
@@ -227,7 +232,35 @@ private List<? extends PlanElement> getRoute(Location origin, Location destinati
         org.matsim.api.core.v01.population.Person matsimPerson = null;
         if (siloPerson != null) {
             matsimPerson = matsimData.getMatsimPopulation().getPersons().get(Id.createPersonId(siloPerson.getId()));
-        }
+
+//			// yyyyyy the following should not be necessary since it is already done upstream, but the containers are not systematically centrally registered.  From here ...
+//
+//			// the following is trying to give the missing vehicle type to the person.  It is nearly (I think) working but now I am getting too tired.
+//			// --> does not work, since one of the calling method is multi-threaded, and at some point two trips of the same person want to insert the same vehicle.
+//
+//			Scenario assembledScenario = matsimData.getScenario();
+//
+//			// create a dummy vehicle type
+//			final Id<VehicleType> defaultVehicleType = Id.create( "defaultVehicleType", VehicleType.class );
+//			VehicleType dummyVehicleType = assembledScenario.getVehicles().getVehicleTypes().get( defaultVehicleType );
+//			final VehiclesFactory vf = assembledScenario.getVehicles().getFactory();
+//			if ( dummyVehicleType==null ) {
+//				dummyVehicleType = vf.createVehicleType( defaultVehicleType );
+//				assembledScenario.getVehicles().addVehicleType(dummyVehicleType);
+//			}
+//
+//			Id<Vehicle> vehicleId = Id.createVehicleId(matsimPerson.getId() );
+//			Vehicle vehicle = assembledScenario.getVehicles().getVehicles().get( vehicleId );
+//			if ( vehicle == null ){
+//				assembledScenario.getVehicles().addVehicle( vf.createVehicle( vehicleId, dummyVehicleType ) );
+//			}
+//			VehicleUtils.insertVehicleIdsIntoAttributes(matsimPerson, Collections.singletonMap( TransportMode.car, vehicleId ) );
+//
+//			// .... to here. yyyyyy
+		}
+
+		// --> The problem is that the vehicle is not in the scenario instance that the tripRouter uses. !!!!
+
         return tripRouter.calcRoute(mode, fromFacility, toFacility, timeOfDay_s, matsimPerson, null);
     }
 

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/SimpleCommuteModeChoiceMatsimScenarioAssembler.java

@@ -40,7 +40,7 @@ public class SimpleCommuteModeChoiceMatsimScenarioAssembler implements MatsimSce
     private final DataContainer dataContainer;
     private final Properties properties;
     private final HandlingOfRandomness handlingOfRandomObject;
-    private CommuteModeChoice commuteModeChoice;
+    private final CommuteModeChoice commuteModeChoice;
     private final Random random;
     // yyyy I found this using the regular silo random number sequence.  In consequence, it was using different random numbers every time it was
     // called, in consequence picking different agents from silo.  This is not what we want.  --  It also picked different agents every time it ran.

extensions/mito2silo/pom.xml

@@ -16,6 +16,7 @@
             <groupId>de.tum.bgu.msm.silo</groupId>
             <artifactId>matsim2silo</artifactId>
             <version>0.1.0-SNAPSHOT</version>
+<!--            <version>0.1.0-20221212.023129-52</version>-->
         </dependency>
     </dependencies>
 

pom.xml

@@ -154,7 +154,9 @@
 
 					<!--execute each test in separate JVM (slow, but otherwise it does not-->
 					<!--work, and no problem as long as there are only integration tests):-->
-					<forkMode>always</forkMode>
+<!--					<forkMode>always</forkMode>-->
+					<forkCount>1</forkCount>
+					<reuseForks>false</reuseForks>
 
 					<!-- avoid out of memory errors: -->
 					<argLine>-Xmx2000m -Djava.awt.headless=true
@@ -177,11 +179,13 @@
 			<artifactId>log4j-core</artifactId>
 			<version>2.24.0</version> <!-- Replace with the correct version -->
 		</dependency>
+
 	</dependencies>
 
 	<properties>
 		<geotools.version>31.3</geotools.version>
-		<matsimVersion>2025.0-PR3643</matsimVersion>
+<!--		<matsimVersion>2025.0-PR3643</matsimVersion>-->
+		<matsimVersion>2025.0</matsimVersion>
 		<jackson.version>2.9.5</jackson.version>
 		<junit.version>5.10.3</junit.version>
 	</properties>

siloCore/pom.xml

@@ -22,6 +22,14 @@
                 <directory>src/main/resources</directory>
             </resource>
         </resources>
+        <extensions>
+            <!--			(chatgpt recommendation for automatic hdf5 native library loading)-->
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>1.7.1</version>
+            </extension>
+        </extensions>
     </build>
 
     <dependencies>
@@ -243,5 +251,12 @@
             <artifactId>xercesImpl</artifactId>
             <version>2.12.0</version>
         </dependency>
+        <dependency>
+            <!--			(chatgpt recommendation for automatic hdf5 native library loading)-->
+            <groupId>org.bytedeco</groupId>
+            <artifactId>hdf5-platform</artifactId>
+            <version>1.14.3-1.5.10</version>
+        </dependency>
+
     </dependencies>
 </project>

siloCore/src/main/java/de/tum/bgu/msm/SiloModel.java

@@ -76,7 +76,9 @@ public void runModel() {
 		long startTime = System.currentTimeMillis();
 		try{
 			setupModel();
+			logger.warn("... done with setupModel, now entering runYearByYear ... ");
 			runYearByYear();
+			logger.warn("... done with runYearByYear, now entering endSimulation ... ");
 			endSimulation();
 		} catch (Exception e){
 			logger.error("Error running SILO.");
@@ -124,7 +126,10 @@ private void runYearByYear() {
 
         final HouseholdDataManager householdDataManager = dataContainer.getHouseholdDataManager();
         for (int year = properties.main.startYear; year < properties.main.endYear; year++) {
-
+			logger.warn("###################################################");
+			logger.warn("###################################################");
+			logger.warn("###################################################");
+			logger.warn("###################################################");
             logger.info("Simulating changes from year " + year + " to year " + (year + 1));
             long time = System.currentTimeMillis();
             SiloUtil.trackingFile("Simulating changes from year " + year + " to year " + (year + 1));

siloCore/src/main/java/de/tum/bgu/msm/models/relocation/migration/InOutMigrationImpl.java

@@ -12,6 +12,7 @@
 import de.tum.bgu.msm.models.autoOwnership.CreateCarOwnershipModel;
 import de.tum.bgu.msm.models.demography.driversLicense.DriversLicenseModel;
 import de.tum.bgu.msm.models.demography.employment.EmploymentModel;
+import de.tum.bgu.msm.models.relocation.moves.MovesModel;
 import de.tum.bgu.msm.models.relocation.moves.MovesModelImpl;
 import de.tum.bgu.msm.properties.Properties;
 import de.tum.bgu.msm.properties.modules.MovesProperties;
@@ -31,7 +32,7 @@ public class InOutMigrationImpl extends AbstractModel implements InOutMigration
     private final static Logger logger = LogManager.getLogger(InOutMigrationImpl.class);
 
     private final EmploymentModel employment;
-    private final MovesModelImpl movesModel;
+    private final MovesModel movesModel;
     private final CreateCarOwnershipModel carOwnership;
     private final DriversLicenseModel driversLicense;
 
@@ -43,9 +44,9 @@ public class InOutMigrationImpl extends AbstractModel implements InOutMigration
     private int lackOfDwellingFailedInmigration;
 
 
-    public InOutMigrationImpl(DataContainer dataContainer, EmploymentModel employment,
-                              MovesModelImpl movesModel, CreateCarOwnershipModel carOwnership,
-                              DriversLicenseModel driversLicense, Properties properties, Random random) {
+    public InOutMigrationImpl( DataContainer dataContainer, EmploymentModel employment,
+							   MovesModel movesModel, CreateCarOwnershipModel carOwnership,
+							   DriversLicenseModel driversLicense, Properties properties, Random random) {
         super(dataContainer, properties, random);
         this.employment = employment;
         this.movesModel = movesModel;