Skip to content

Output failed tests in execution order #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ultrasanic wants to merge 3 commits into
base: 2025
Choose a base branch
from
Open

Output failed tests in execution order #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
275def6
Output failed tests in execution order
Ultrasanic Dec 22, 2025
1c7e223
Add indexmap to Cargo.toml
Ultrasanic Dec 24, 2025
7e77a1b
Output failed tests in execution order within each group
Ultrasanic Dec 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions homework/01-guarantees/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ anysystem = "0.1.2"
assertables = "3.2.2"
clap = { version = "3.1.17", features = ["cargo", "derive"] }
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.14"
rand = "0.8.5"
rand_pcg = "0.3.1"
Expand Down
5 changes: 3 additions & 2 deletions homework/01-guarantees/tests/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ mod common;
mod tests;
mod tests_mc;

use std::collections::{BTreeMap, HashSet};
use std::collections::HashSet;
use std::env;
use std::io::Write;

use clap::Parser;
use env_logger::Builder;
use indexmap::IndexMap;
use log::LevelFilter;

use anysystem::test::{TestResult, TestSuite};
Expand Down Expand Up @@ -272,7 +273,7 @@ fn main() {
}
}

fn score(results: BTreeMap<String, TestResult>) -> f32 {
fn score(results: IndexMap<String, TestResult>) -> f32 {
let guarantees = HashSet::from(["AT MOST ONCE", "AT LEAST ONCE", "EXACTLY ONCE", "EXACTLY ONCE ORDERED"]);
let mut failed_guarantees: HashSet<&str> = HashSet::new();
let mut failed_overheads: HashSet<&str> = HashSet::new();
Expand Down
1 change: 1 addition & 0 deletions homework/04-broadcast/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ edition = "2021"
anysystem = "0.1.2"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.17"
rand = "0.8.5"
rand_pcg = "0.3.1"
Expand Down
5 changes: 3 additions & 2 deletions homework/04-broadcast/tests/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ mod common;
mod tests;
mod tests_mc;

use std::collections::{BTreeMap, HashSet};
use std::collections::HashSet;
use std::env;
use std::io::Write;

use clap::Parser;
use env_logger::Builder;
use indexmap::IndexMap;
use log::LevelFilter;

use anysystem::python::PyProcessFactory;
Expand Down Expand Up @@ -99,7 +100,7 @@ fn main() {
}
}

fn score(results: BTreeMap<String, TestResult>) -> f32 {
fn score(results: IndexMap<String, TestResult>) -> f32 {
let mut violated = HashSet::new();
for (_, result) in results {
if let Err(e) = result {
Expand Down
1 change: 1 addition & 0 deletions homework/06-membership/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ anysystem = "0.1.2"
assertables = "3.2.2"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.17"
rand = "0.8.5"
rand_pcg = "0.3.1"
Expand Down
24 changes: 10 additions & 14 deletions homework/06-membership/tests/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ mod common;
mod tests;
mod tests_mc;

use std::collections::{BTreeMap, HashSet};
use std::env;
use std::io::Write;

use clap::Parser;
use env_logger::Builder;
use indexmap::{IndexMap, IndexSet};
use log::LevelFilter;
use rand::prelude::*;
use rand_pcg::Pcg64;
Expand Down Expand Up @@ -148,11 +148,11 @@ fn main() {

macro_rules! string_set {
($($x:expr),+ $(,)?) => (
HashSet::from([$($x.to_string()),+])
IndexSet::from([$($x.to_string()),+])
);
}

fn score(results: BTreeMap<String, TestResult>, monkeys: u32, disable_mc_tests: bool) -> f32 {
fn score(results: IndexMap<String, TestResult>, monkeys: u32, disable_mc_tests: bool) -> f32 {
let basic_group = string_set! {
"SIMPLE",
"GET MEMBERS SEMANTICS",
Expand Down Expand Up @@ -188,11 +188,11 @@ fn score(results: BTreeMap<String, TestResult>, monkeys: u32, disable_mc_tests:
let mut all_groups =
&(&(&basic_group | &process_crash_group) | &network_failure_group) | &scalability_group;
if disable_mc_tests {
all_groups.remove("MODEL CHECKING");
all_groups.shift_remove("MODEL CHECKING");
}
assert_eq!(all_groups, results.keys().cloned().collect());
assert_eq!(all_groups, results.keys().cloned().collect::<IndexSet<String>>());

let mut passed = HashSet::new();
let mut passed = IndexSet::new();
let mut model_checking_ok = false;
for (test, result) in results {
match result {
Expand All @@ -216,35 +216,31 @@ fn score(results: BTreeMap<String, TestResult>, monkeys: u32, disable_mc_tests:
score += 4.;
println!("passed");
} else {
let mut failed: Vec<&String> = basic_group.difference(&passed).collect();
failed.sort();
let failed: Vec<&String> = basic_group.difference(&passed).collect();
println!("not passed {failed:?}");
}
print!("Process crashes tests: ");
if process_crash_group.is_subset(&passed) {
score += 3.;
println!("passed");
} else {
let mut failed: Vec<&String> = process_crash_group.difference(&passed).collect();
failed.sort();
let failed: Vec<&String> = process_crash_group.difference(&passed).collect();
println!("not passed {failed:?}");
}
print!("Network failures tests: ");
if network_failure_group.is_subset(&passed) {
score += 3.;
println!("passed");
} else {
let mut failed: Vec<&String> = network_failure_group.difference(&passed).collect();
failed.sort();
let failed: Vec<&String> = network_failure_group.difference(&passed).collect();
println!("not passed {failed:?}");
}
print!("Scalability tests: ");
if scalability_group.is_subset(&passed) {
score += 4.;
println!("passed");
} else {
let mut failed: Vec<&String> = scalability_group.difference(&passed).collect();
failed.sort();
let failed: Vec<&String> = scalability_group.difference(&passed).collect();
println!("not passed {failed:?}");
}
print!("Model checking test: ");
Expand Down
1 change: 1 addition & 0 deletions homework/07-kv-sharding/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ assertables = "3.2.2"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
decorum = "0.3.1"
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.17"
rand = "0.8.5"
rand_pcg = "0.3.1"
Expand Down
10 changes: 5 additions & 5 deletions homework/07-kv-sharding/tests/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ mod common;
mod tests;
mod tests_mc;

use std::collections::{BTreeMap, HashSet};
use std::env;
use std::io::Write;

use clap::Parser;
use env_logger::Builder;
use indexmap::{IndexMap, IndexSet};
use log::LevelFilter;

use anysystem::python::PyProcessFactory;
Expand Down Expand Up @@ -108,11 +108,11 @@ fn main() {

macro_rules! string_set {
($($x:expr),+ $(,)?) => (
HashSet::from([$($x.to_string()),+])
IndexSet::from([$($x.to_string()),+])
);
}

fn score(results: BTreeMap<String, TestResult>) -> f32 {
fn score(results: IndexMap<String, TestResult>) -> f32 {
let basic_group = string_set! {
"SINGLE NODE", "INSERTS", "DELETES", "MEMORY OVERHEAD", "MC NORMAL"
};
Expand All @@ -124,9 +124,9 @@ fn score(results: BTreeMap<String, TestResult>) -> f32 {
"DISTRIBUTION", "DISTRIBUTION NODE ADDED", "DISTRIBUTION NODE REMOVED"
};
let all_groups = &(&basic_group | &node_changes_group) | &distribution_group;
assert_eq!(all_groups, results.keys().cloned().collect());
assert_eq!(all_groups, results.keys().cloned().collect::<IndexSet<String>>());

let mut passed = HashSet::new();
let mut passed = IndexSet::new();
for (test, result) in results {
if result.is_ok() {
passed.insert(test);
Expand Down
1 change: 1 addition & 0 deletions homework/08-kv-replication/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ byteorder = "1.4.3"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
decorum = "0.3.1"
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.17"
md5 = "0.7.0"
rand = "0.8.5"
Expand Down
10 changes: 5 additions & 5 deletions homework/08-kv-replication/tests/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ mod common;
mod tests;
mod tests_mc;

use std::collections::{BTreeMap, HashSet};
use std::env;
use std::io::Write;

use clap::Parser;
use env_logger::Builder;
use indexmap::{IndexMap, IndexSet};
use log::LevelFilter;

use anysystem::python::PyProcessFactory;
Expand Down Expand Up @@ -93,11 +93,11 @@ fn main() {

macro_rules! string_set {
($($x:expr),+ $(,)?) => (
HashSet::from([$($x.to_string()),+])
IndexSet::from([$($x.to_string()),+])
);
}

fn score(results: BTreeMap<String, TestResult>) -> f32 {
fn score(results: IndexMap<String, TestResult>) -> f32 {
let basic_group = string_set! {
"BASIC", "REPLICAS CHECK", "CONCURRENT GET PUT", "MC BASIC"
};
Expand All @@ -110,9 +110,9 @@ fn score(results: BTreeMap<String, TestResult>) -> f32 {
"PARTITION CLIENTS", "PARTITION MIXED"
};
let all_groups = &(&basic_group | &conflict_resolution_group) | &sloppy_partition_group;
assert_eq!(all_groups, results.keys().cloned().collect());
assert_eq!(all_groups, results.keys().cloned().collect::<IndexSet<String>>());

let mut passed = HashSet::new();
let mut passed = IndexSet::new();
for (test, result) in results {
if result.is_ok() {
passed.insert(test);
Expand Down
1 change: 1 addition & 0 deletions homework/09-kv-replication-2/tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ byteorder = "1.4.3"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
decorum = "0.3.1"
env_logger = "0.9.0"
indexmap = "2.12.1"
log = "0.4.17"
md5 = "0.7.0"
rand = "0.8.5"
Expand Down
Loading