diff --git a/docs/supported-mutators.md b/docs/supported-mutators.md index a4e4a25d4..afa0339de 100644 --- a/docs/supported-mutators.md +++ b/docs/supported-mutators.md @@ -28,39 +28,41 @@ All Stryker versions support a variety of different mutators. We've aligned on a ## Arithmetic Operator -| Original | Mutated | -| -------- | ------- | -| `a + b` | `a - b` | -| `a - b` | `a + b` | -| `a * b` | `a / b` | -| `a / b` | `a * b` | -| `a % b` | `a * b` | +| Mutant operator | Original | Mutated | +| -------------------------------------------- | -------- | ------- | +| AdditionOperatorNegation | `a + b` | `a - b` | +| SubtractionOperatorNegation | `a - b` | `a + b` | +| MultiplicationOperatorNegation | `a * b` | `a / b` | +| DivisionOperatorNegation | `a / b` | `a * b` | +| RemainderOperatorToMultiplicationReplacement | `a % b` | `a * b` | [๐Ÿ” Back to Top](#) ## Array Declaration -| Original | Mutated | -| ----------------------- | ------------- | -| `new Array(1, 2, 3, 4)` | `new Array()` | -| `[1, 2, 3, 4]` | `[ ]` | +| Mutant operator | Original | Mutated | +| ---------------------------- | ----------------------- | ------------------------------- | +| ArrayConstructorItemsFill | `new Array()` | `new Array('Stryker was here')` | +| ArrayConstructorItemsRemoval | `new Array(1, 2, 3, 4)` | `new Array()` | +| ArrayLiteralItemsFill | `[ ]` | `['Stryker was here']` | +| ArrayLiteralItemsRemoval | `[1, 2, 3, 4]` | `[ ]` | [๐Ÿ” Back to Top](#) ## Assignment Expression -| Original | Mutated | -| -------------------- | -------------------- | -| `+=` | `-=` | -| `-=` | `+=` | -| `*=` | `/=` | -| `/=` | `*=` | -| `%=` | `*=` | -| `<<=` | `>>=` | -| `>>=` | `<<=` | -| `&=` | |= | -| |= | `&=` | -| `??=` | `&&=`ยน | +| Mutant operator | Original | Mutated | +| ---------------------------------------------------------- | -------------------- | -------------------- | +| AdditionAssignmentNegation | `+=` | `-=` | +| SubtractionAssignmentNegation | `-=` | `+=` | +| MultiplicationAssignmentNegation | `*=` | `/=` | +| DivisionAssignmentNegation | `/=` | `*=` | +| RemainderAssignmentToMultiplicationReplacement | `%=` | `*=` | +| LeftShiftAssignmentNegation | `<<=` | `>>=` | +| RightShiftAssignmentNegation | `>>=` | `<<=` | +| BitwiseAndAssignmentToBitwiseOrReplacement | `&=` | |= | +| BitwiseOrAssignmentToBitwiseAndReplacement | |= | `&=` | +| NullishCoalescingOperatorToLogicalAndAssignmentReplacement | `??=` | `&&=`ยน | - ยน: Only supported by Stryker-JS @@ -86,11 +88,11 @@ function saySomething() {} ## Boolean Literal -| Original | Mutated | -| ----------- | ---------- | -| `true` | `false` | -| `false` | `true` | -| `!(a == b)` | `a == b` ยน | +| Mutant operator | Original | Mutated | +| -------------------- | ----------- | ---------- | +| TrueLiteralNegation | `true` | `false` | +| FalseLiteralNegation | `false` | `true` | +| LogicalNotRemoval | `!(a == b)` | `a == b` ยน | - ยน: Not supported by Stryker4s @@ -100,23 +102,24 @@ function saySomething() {} Stryker.NET _specific mutator_ -| Original | Mutated | -| ---------------- | ------- | -| `checked(2 + 4)` | `2 + 4` | +| Mutant operator | Original | Mutated | +| ------------------------ | ---------------- | ------- | +| CheckedExpressionRemoval | `checked(2 + 4)` | `2 + 4` | [๐Ÿ” Back to Top](#) ## Conditional Expression -| Original | Mutated | -| ---------------------------------- | ----------------------------------- | -| `for (var i = 0; i < 10; i++) { }` | `for (var i = 0; false; i++) { }` ยน | -| `while (a > b) { }` | `while (false) { }` | -| `do { } while (a > b);` | `do { } while (false);` | -| `if (a > b) { }` | `if (true) { }` | -| `if (a > b) { }` | `if (false) { }` | -| `var x = a > b ? 1 : 2;` | `var x = true ? 1 : 2;` ยน | -| `var x = a > b ? 1 : 2;` | `var x = false ? 1 : 2;` ยน | +| Mutant operator | Original | Mutated | +| -------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- | +| ForLoopConditionToFalseReplacement | `for (var i = 0; i < 10; i++) { }` | `for (var i = 0; false; i++) { }` ยน | +| WhileLoopConditionToFalseReplacement | `while (a > b) { }` | `while (false) { }` | +| DoWhileLoopConditionToFalseReplacement | `do { } while (a > b);` | `do { } while (false);` | +| IfConditionToTrueReplacement | `if (a > b) { }` | `if (true) { }` | +| IfConditionToFalseReplacement | `if (a > b) { }` | `if (false) { }` | +| BooleanExpressionToTrueReplacement | `var x = a > b ? 1 : 2;` | `var x = true ? 1 : 2;` ยน | +| BooleanExpressionToFalseReplacement | `var x = a > b ? 1 : 2;` | `var x = false ? 1 : 2;` ยน | +| SwitchStatementBodyRemoval | `switch(x) { case 1: doSomething(); default: default(); }` | `switch(x) { case 1: default: default(); }` | - ยน: Not supported by Stryker4s @@ -124,20 +127,20 @@ Stryker.NET _specific mutator_ ## Equality Operator -| Original | Mutated | -| --------- | ----------- | -| `a < b` | `a <= b` | -| `a < b` | `a >= b` | -| `a <= b` | `a < b` | -| `a <= b` | `a > b` | -| `a > b` | `a >= b` | -| `a > b` | `a <= b` | -| `a >= b` | `a > b` | -| `a >= b` | `a < b` | -| `a == b` | `a != b` | -| `a != b` | `a == b` | -| `a === b` | `a !== b` ยน | -| `a !== b` | `a === b` ยน | +| Mutant operator | Original | Mutated | +| -------------------------------- | --------- | ----------- | +| LessThanOperatorBoundary | `a < b` | `a <= b` | +| LessThanOperatorNegation | `a < b` | `a >= b` | +| LessThanEqualOperatorBoundary | `a <= b` | `a < b` | +| LessThanEqualOperatorNegation | `a <= b` | `a > b` | +| GreaterThanOperatorBoundary | `a > b` | `a >= b` | +| GreaterThanOperatorNegation | `a > b` | `a <= b` | +| GreaterThanEqualOperatorBoundary | `a >= b` | `a > b` | +| GreaterThanEqualOperatorNegation | `a >= b` | `a < b` | +| EqualityOperatorNegation | `a == b` | `a != b` | +| InequalityOperatorNegation | `a != b` | `a == b` | +| StrictEqualityOperatorNegation | `a === b` | `a !== b` ยน | +| StrictInequalityOperatorNegation | `a !== b` | `a === b` ยน | - ยน: Only supported on StrykerJS and Stryker4s @@ -145,11 +148,11 @@ Stryker.NET _specific mutator_ ## Logical Operator -| Original | Mutated | -| ----------------------------- | ----------------------------- | -| `a && b` | a || b | -| a || b | `a && b` | -| `a ?? b` | `a && b` | +| Mutant operator | Original | Mutated | +| ------------------------------------------------ | ----------------------------- | ----------------------------- | +| LogicalAndOperatorToLogicalOrReplacement | `a && b` | a || b | +| LogicalOrOperatorToLogicalAndReplacement | a || b | `a && b` | +| NullishCoalescingOperatorToLogicalAndReplacement | `a ?? b` | `a && b` | [๐Ÿ” Back to Top](#) @@ -159,92 +162,92 @@ Due to differences in language syntax, method expressions are implemented differ ### StrykerJS -| Original | Mutated | -| -------------------- | --------------------- | -| `endsWith()` | `startsWith()` | -| `startsWith()` | `endsWith()` | -| `trim()` | ` ` | -| `trimEnd()` | `trimStart()` | -| `trimStart()` | `trimEnd()` | -| `substr()` | ` ` | -| `substring()` | ` ` | -| `toUpperCase()` | ` toLowerCase()` | -| `toLowerCase()` | ` toUpperCase()` | -| `toLocalLowerCase()` | `toLocalUpperCase()` | -| `toLocalUpperCase()` | ` toLocalLowerCase()` | -| `sort()` | ` ` | -| `some()` | `every()` | -| `every()` | `some()` | -| `reverse()` | ` ` | -| `filter()` | ` ` | -| `slice()` | ` ` | -| `charAt()` | ` ` | -| `min()` | `max()` | -| `max()` | `min()` | +| Mutant operator | Original | Mutated | +| ----------------------------------- | -------------------- | --------------------- | +| EndsWithMethodCallNegation | `endsWith()` | `startsWith()` | +| StartsWithMethodCallNegation | `startsWith()` | `endsWith()` | +| TrimMethodCallRemoval | `trim()` | ` ` | +| TrimEndMethodCallNegation | `trimEnd()` | `trimStart()` | +| TrimStartMethodCallNegation | `trimStart()` | `trimEnd()` | +| SubstrMethodCallRemoval | `substr()` | ` ` | +| SubstringMethodCallRemoval | `substring()` | ` ` | +| ToUpperCaseMethodCallNegation | `toUpperCase()` | ` toLowerCase()` | +| ToLowerCaseMethodCallNegation | `toLowerCase()` | ` toUpperCase()` | +| ToLocaleLowerCaseMethodCallNegation | `toLocalLowerCase()` | `toLocalUpperCase()` | +| ToLocaleUpperCaseMethodCallNegation | `toLocalUpperCase()` | ` toLocalLowerCase()` | +| SortMethodCallRemoval | `sort()` | ` ` | +| SomeMethodCallToEveryReplacement | `some()` | `every()` | +| EveryMethodCallToSomeReplacement | `every()` | `some()` | +| ReverseMethodCallRemoval | `reverse()` | ` ` | +| FilterMethodCallRemoval | `filter()` | ` ` | +| SliceMethodCallRemoval | `slice()` | ` ` | +| CharAtMethodCallRemoval | `charAt()` | ` ` | +| MinMethodCallNegation | `min()` | `max()` | +| MaxMethodCallNegation | `max()` | `min()` | ### Stryker.NET -| Original | Mutated | -| --------------------- | ------------------- | -| `Distinct()` | ` ` | -| `Reverse()` | ` ` | -| `OrderBy()` | ` ` | -| `OrderByDescending()` | ` ` | -| `SingleOrDefault()` | `FirstOrDefault()` | -| `FirstOrDefault()` | `SingleOrDefault()` | -| `First()` | `Last()` | -| `Last()` | `First()` | -| `All()` | `Any()` | -| `Any()` | `All()` | -| `Skip()` | `Take()` | -| `Take()` | `Skip()` | -| `SkipWhile()` | `TakeWhile()` | -| `TakeWhile()` | `SkipWhile()` | -| `Min()` | `Max()` | -| `Max()` | `Min()` | -| `Sum()` | `Count()` | -| `Count()` | `Sum()` | +| Mutant operator | Original | Mutated | +| ---------------------------------------------------- | --------------------- | ------------------- | +| DistinctMethodCallRemoval | `Distinct()` | ` ` | +| ReverseMethodCallRemoval | `Reverse()` | ` ` | +| OrderByMethodCallRemoval | `OrderBy()` | ` ` | +| OrderByDescendingMethodCallRemoval | `OrderByDescending()` | ` ` | +| SingleOrDefaultMethodCallToFirstOrDefaultReplacement | `SingleOrDefault()` | `FirstOrDefault()` | +| FirstOrDefaultMethodCallToSingleOrDefaultReplacement | `FirstOrDefault()` | `SingleOrDefault()` | +| FirstMethodCallNegation | `First()` | `Last()` | +| LastMethodCallNegation | `Last()` | `First()` | +| AllMethodCallToAnyReplacement | `All()` | `Any()` | +| AnyMethodCallToAllReplacement | `Any()` | `All()` | +| SkipMethodCallNegation | `Skip()` | `Take()` | +| TakeMethodCallNegation | `Take()` | `Skip()` | +| SkipWhileMethodCallNegation | `SkipWhile()` | `TakeWhile()` | +| TakeWhileMethodCallNegation | `TakeWhile()` | `SkipWhile()` | +| MinMethodCallNegation | `Min()` | `Max()` | +| MaxMethodCallNegation | `Max()` | `Min()` | +| SumMethodCallToCountReplacement | `Sum()` | `Count()` | +| CountMethodCallToSumReplacement | `Count()` | `Sum()` | ### Stryker4s -| Original | Mutated | -| ------------------ | ------------------ | -| `a.filter(b)` | `a.filterNot(b)` | -| `a.filterNot(b)` | `a.filter(b)` | -| `a.exists(b)` | `a.forall(b)` | -| `a.forall(b)` | `a.exists(b)` | -| `a.take(b)` | `a.drop(b)` | -| `a.drop(b)` | `a.take(b)` | -| `a.takeRight(b)` | `a.dropRight(b)` | -| `a.dropRight(b)` | `a.takeRight(b)` | -| `a.takeWhile(b)` | `a.dropWhile(b)` | -| `a.dropWhile(b)` | `a.takeWhile(b)` | -| `a.isEmpty` | `a.nonEmpty` | -| `a.nonEmpty` | `a.isEmpty` | -| `a.indexOf` | `a.lastIndexOf(b)` | -| `a.lastIndexOf(b)` | `a.indexOf(b)` | -| `a.max` | `a.min` | -| `a.min` | `a.max` | -| `a.maxBy(b)` | `a.minBy(b)` | -| `a.minBy(b)` | `a.maxBy(b)` | +| Mutant operator | Original | Mutated | +| ----------------------------------------- | ------------------ | ------------------ | +| FilterMethodCallNegation | `a.filter(b)` | `a.filterNot(b)` | +| FilterNotMethodCallNegation | `a.filterNot(b)` | `a.filter(b)` | +| ExistsMethodCallToForAllReplacement | `a.exists(b)` | `a.forall(b)` | +| ForAllMethodCallToExistsReplacement | `a.forall(b)` | `a.exists(b)` | +| TakeMethodCallNegation | `a.take(b)` | `a.drop(b)` | +| DropMethodCallNegation | `a.drop(b)` | `a.take(b)` | +| TakeRightMethodCallNegation | `a.takeRight(b)` | `a.dropRight(b)` | +| DropRightMethodCallNegation | `a.dropRight(b)` | `a.takeRight(b)` | +| TakeWhileMethodCallNegation | `a.takeWhile(b)` | `a.dropWhile(b)` | +| DropWhileMethodCallNegation | `a.dropWhile(b)` | `a.takeWhile(b)` | +| IsEmptyMethodCallNegation | `a.isEmpty` | `a.nonEmpty` | +| IsNonEmptyMethodCallNegation | `a.nonEmpty` | `a.isEmpty` | +| IndexOfMethodCallToLastIndexOfReplacement | `a.indexOf` | `a.lastIndexOf(b)` | +| LastIndexOfMethodCallToIndexOfReplacement | `a.lastIndexOf(b)` | `a.indexOf(b)` | +| MaxMethodCallNegation | `a.max` | `a.min` | +| MinMethodCallNegation | `a.min` | `a.max` | +| MaxByMethodCallNegation | `a.maxBy(b)` | `a.minBy(b)` | +| MinByMethodCallNegation | `a.minBy(b)` | `a.maxBy(b)` | [๐Ÿ” Back to Top](#) ## Object literal -| Original | Mutated | -| ---------------- | ------- | -| `{ foo: 'bar' }` | `{ }` | +| Mutant operator | Original | Mutated | +| ------------------------------ | ---------------- | ------- | +| ObjectLiteralPropertiesRemoval | `{ foo: 'bar' }` | `{ }` | [๐Ÿ” Back to Top](#) ## Optional chaining -| Original | Mutated | -| ---------- | --------- | -| `foo?.bar` | `foo.bar` | -| `foo?.[1]` | `foo[1]` | -| `foo?.()` | `foo()` | +| Mutant operator | Original | Mutated | +| ----------------------------------------------- | ---------- | --------- | +| OptionalMemberExpressionOptionalRemoval | `foo?.bar` | `foo.bar` | +| OptionalComputedMemberExpressionOptionalRemoval | `foo?.[1]` | `foo[1]` | +| OptionalCallExpressionOptionalRemoval | `foo?.()` | `foo()` | [๐Ÿ” Back to Top](#) @@ -293,11 +296,12 @@ Strings and literals identified to a regex are mutated in the following way: ## String Literal -| Original | Mutated | -| -------------------------------------- | --------------------- | -| `"foo"` (non-empty string) | `""` (empty string) | -| `""` (empty string) | `"Stryker was here!"` | -| `s"foo ${bar}"` (string interpolation) | `s""` ยน | +| Mutant operator | Original | Mutated | +| ------------------------------------------ | -------------------------------------- | --------------------- | +| FilledStringLiteralToEmptyReplacement | `"foo"` (filled string) | `""` (empty string) | +| EmptyStringLiteralToFilledReplacement | `""` (empty string) | `"Stryker was here!"` | +| FilledInterpolatedStringToEmptyReplacement | `s"foo ${bar}"` (string interpolation) | `s""` ยน | +| FilledInterpolatedStringToEmptyReplacement | `\`foo ${bar}\`` | `\`\`` | ยน For Stryker4s, only works with string interpolation and not other types of interpolation (like [Scalameta quasiquotes](https://scalameta.org/docs/trees/guide.html#with-quasiquotes)) to avoid compile errors @@ -305,20 +309,21 @@ Strings and literals identified to a regex are mutated in the following way: ## Unary Operator -| Original | Mutated | -| -------- | ------- | -| `+a` | `-a` | -| `-a` | `+a` | +| Mutant operator | Original | Mutated | +| ------------------------- | -------- | ------- | +| UnaryBitwiseNotRemoval | `~a` | `a` | +| UnaryPlusOperatorNegation | `+a` | `-a` | +| UnaryMinOperatorNegation | `-a` | `+a` | [๐Ÿ” Back to Top](#) ## Update Operator -| Original | Mutated | -| -------- | ------- | -| `a++` | `a--` | -| `a--` | `a++` | -| `++a` | `--a` | -| `--a` | `++a` | +| Mutant operator | Original | Mutated | +| -------------------------------- | -------- | ------- | +| PostfixIncrementOperatorNegation | `a++` | `a--` | +| PostfixDecrementOperatorNegation | `a--` | `a++` | +| PrefixIncrementOperatorNegation | `++a` | `--a` | +| PostfixDecrementOperatorNegation | `--a` | `++a` | [๐Ÿ” Back to Top](#) diff --git a/packages/elements/test/unit/helpers/factory.ts b/packages/elements/test/unit/helpers/factory.ts index 870bb76a9..18dd81bfa 100644 --- a/packages/elements/test/unit/helpers/factory.ts +++ b/packages/elements/test/unit/helpers/factory.ts @@ -99,6 +99,7 @@ export function createMetrics(overrides?: Metrics): Metrics { const defaults: Metrics = { pending: 0, killed: 0, + ignoredByMutationLevel: 0, survived: 0, timeout: 0, compileErrors: 0, @@ -112,6 +113,7 @@ export function createMetrics(overrides?: Metrics): Metrics { totalUndetected: 0, totalValid: 0, mutationScore: 0, + adjustedMutationScore: 0, mutationScoreBasedOnCoveredCode: 0, }; return { ...defaults, ...overrides }; diff --git a/packages/metrics/src/calculateMetrics.ts b/packages/metrics/src/calculateMetrics.ts index f58b09a61..b30a2a412 100644 --- a/packages/metrics/src/calculateMetrics.ts +++ b/packages/metrics/src/calculateMetrics.ts @@ -7,6 +7,7 @@ import { TestStatus } from './model/test-model.js'; const DEFAULT_SCORE = NaN; const ROOT_NAME = 'All files'; const ROOT_NAME_TESTS = 'All tests'; +const IGNORED_BY_LEVEL_STATUS = 'Ignored by level'; /** * Calculates the files-under-test metrics inside of a mutation testing report @@ -135,6 +136,7 @@ export function countTestFileMetrics(testFile: TestFileModel[]): TestMetrics { export function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { const mutants = fileResult.flatMap((_) => _.mutants); + const ignoredByMutationLevel = mutants.filter((_) => _.status === 'Ignored' && _.statusReason === IGNORED_BY_LEVEL_STATUS).length; const count = (status: MutantStatus) => mutants.filter((_) => _.status === status).length; const pending = count('Pending'); const killed = count('Killed'); @@ -149,6 +151,10 @@ export function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { const totalCovered = totalDetected + survived; const totalValid = totalUndetected + totalDetected; const totalInvalid = runtimeErrors + compileErrors; + const mutationScore = totalValid > 0 ? (totalDetected / totalValid) * 100 : DEFAULT_SCORE; + const totalMutants = totalValid + totalInvalid + ignored + pending; + const mutationScoreBasedOnCoveredCode = totalValid > 0 ? (totalDetected / totalCovered) * 100 || 0 : DEFAULT_SCORE; + const adjustedMutationScore = (mutationScore * (totalMutants - ignoredByMutationLevel)) / totalMutants; return { pending, killed, @@ -158,13 +164,15 @@ export function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { runtimeErrors, compileErrors, ignored, + ignoredByMutationLevel, totalDetected, totalUndetected, totalCovered, totalValid, totalInvalid, - mutationScore: totalValid > 0 ? (totalDetected / totalValid) * 100 : DEFAULT_SCORE, - totalMutants: totalValid + totalInvalid + ignored + pending, - mutationScoreBasedOnCoveredCode: totalValid > 0 ? (totalDetected / totalCovered) * 100 || 0 : DEFAULT_SCORE, + mutationScore, + totalMutants, + mutationScoreBasedOnCoveredCode, + adjustedMutationScore, }; } diff --git a/packages/metrics/src/model/metrics.ts b/packages/metrics/src/model/metrics.ts index 2ce5e87b8..4e530a35e 100644 --- a/packages/metrics/src/model/metrics.ts +++ b/packages/metrics/src/model/metrics.ts @@ -36,6 +36,10 @@ export interface Metrics { * The total number of mutants that were not even tested because the config of the user asked for them to be ignored. */ ignored: number; + /** + * The number of mutants that were not tested because they were not included in the specified mutation level. + */ + ignoredByMutationLevel: number; /** * The total number of mutants that were detected, meaning either killed or caused a time out. * `killed + timed out` @@ -76,4 +80,9 @@ export interface Metrics { * `totalDetected / totalCovered * 100` */ mutationScoreBasedOnCoveredCode: number; + /** + * An estimate for the mutation score, based upon the mutation score that was achieved with the Mutation Level. + * `mutationScore * (totalMutants - ignoredByMutationLevel) / totalMutants` + */ + adjustedMutationScore: number; }