diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt new file mode 100644 index 0000000000..afeb1edb24 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt @@ -0,0 +1,92 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.* + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/resources"]) +@RestController +open class FailModificationApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FailModificationApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset(){ + data.clear() + } + } + + data class ResourceData( + var name: String, + var value: Int + ) + + data class UpdateRequest( + val name: String?, + val value: Int? + ) + + + @PostMapping + open fun create(@RequestBody body: ResourceData): ResponseEntity { + val id = data.size + 1 + data[id] = body.copy() + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/{id}"]) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/{id}"]) + open fun put( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // bug: modifies data even though it will return 4xx + if(body.name != null) { + resource.name = body.name + } + if(body.value != null) { + resource.value = body.value + } + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/{id}"]) + open fun patch( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + if(body.name == null && body.value == null) { + return ResponseEntity.status(400).body("No fields to update") + } + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } + +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt new file mode 100644 index 0000000000..a507132fb1 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt @@ -0,0 +1,11 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController + + +class FailModificationController: SpringController(FailModificationApplication::class.java){ + + override fun resetStateOfSUT() { + FailModificationApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt new file mode 100644 index 0000000000..5fa5a8b617 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -0,0 +1,46 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FailModificationEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "FailedModificationEM", + 500 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first()) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 2ffccf9836..eb510cb72c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -35,6 +35,8 @@ enum class ExperimentalFaultCategory( "TODO"), HTTP_REPEATED_CREATE_PUT(914, "Repeated PUT Creates Resource With 201", "repeatedCreatePut", "TODO"), + HTTP_SIDE_EFFECTS_FAILED_MODIFICATION(915, "A failed PUT or PATCH must not change the resource", "sideEffectsFailedModification", + "TODO"), //3xx: GraphQL diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index c934b40862..30cf97315d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -1,10 +1,14 @@ package org.evomaster.core.problem.rest.oracle +import com.google.gson.JsonParser import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.rest.StatusGroup import org.evomaster.core.search.action.ActionResult +import org.evomaster.core.search.gene.ObjectGene object HttpSemanticsOracle { @@ -103,4 +107,141 @@ object HttpSemanticsOracle { return NonWorkingDeleteResult(checkingDelete, nonWorking, delete.getName(), actions.size - 2) } -} \ No newline at end of file + + fun hasSideEffectFailedModification(individual: RestIndividual, + actionResults: List + ): Boolean{ + + if(individual.size() < 3){ + return false + } + + val actions = individual.seeMainExecutableActions() + + val before = actions[actions.size - 3] // GET (before state) + val modify = actions[actions.size - 2] // PUT or PATCH (failed modification) + val after = actions[actions.size - 1] // GET (after state) + + // check verbs: GET, PUT|PATCH, GET + if(before.verb != HttpVerb.GET) { + return false + } + if(modify.verb != HttpVerb.PUT && modify.verb != HttpVerb.PATCH) { + return false + } + if(after.verb != HttpVerb.GET) { + return false + } + + // all three must be on the same resolved path + if(!before.usingSameResolvedPath(modify) || !after.usingSameResolvedPath(modify)) { + return false + } + + // auth should be consistent + if(before.auth.isDifferentFrom(modify.auth) || after.auth.isDifferentFrom(modify.auth)) { + return false + } + + val resBefore = actionResults.find { it.sourceLocalId == before.getLocalId() } as RestCallResult? + ?: return false + val resModify = actionResults.find { it.sourceLocalId == modify.getLocalId() } as RestCallResult? + ?: return false + val resAfter = actionResults.find { it.sourceLocalId == after.getLocalId() } as RestCallResult? + ?: return false + + // before GET must be 2xx + if(!StatusGroup.G_2xx.isInGroup(resBefore.getStatusCode())) { + return false + } + + // PUT/PATCH must have failed with 4xx + if(!StatusGroup.G_4xx.isInGroup(resModify.getStatusCode())) { + return false + } + + // after GET must be 2xx + if(!StatusGroup.G_2xx.isInGroup(resAfter.getStatusCode())) { + return false + } + + val bodyBefore = resBefore.getBody() + val bodyAfter = resAfter.getBody() + + // if both are null/empty, no side-effect detected + if(bodyBefore.isNullOrEmpty() && bodyAfter.isNullOrEmpty()) { + return false + } + + // extract the field names sent in the PUT/PATCH request body + val modifiedFieldNames = extractModifiedFieldNames(modify) + + // if we can identify specific fields, compare only those to avoid false positives from timestamps etc. + if(modifiedFieldNames.isNotEmpty() + && !bodyBefore.isNullOrEmpty() + && !bodyAfter.isNullOrEmpty()) { + return hasChangedModifiedFields(bodyBefore, bodyAfter, modifiedFieldNames) + } + + // otherwise compare entire bodies + return bodyBefore != bodyAfter + } + + /** + * Extract field names from the PUT/PATCH request body. + * These are the fields that the client attempted to modify. + */ + private fun extractModifiedFieldNames(modify: RestCallAction): Set { + + val bodyParam = modify.parameters.find { it is BodyParam } as BodyParam? + ?: return emptySet() + + val gene = bodyParam.primaryGene() + val objectGene = gene.getWrappedGene(ObjectGene::class.java) as ObjectGene? + ?: if (gene is ObjectGene) gene else null + + if(objectGene == null){ + return emptySet() + } + + return objectGene.fields.map { it.name }.toSet() + } + + /** + * Compare only the fields that were sent in the PUT/PATCH request. + * Returns true if any of those fields changed between before and after GET responses. + */ + internal fun hasChangedModifiedFields( + bodyBefore: String, + bodyAfter: String, + fieldNames: Set + ): Boolean { + + try { + val jsonBefore = JsonParser.parseString(bodyBefore) + val jsonAfter = JsonParser.parseString(bodyAfter) + + if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject){ + // not JSON objects, fallback to full comparison + return bodyBefore != bodyAfter + } + + val objBefore = jsonBefore.asJsonObject + val objAfter = jsonAfter.asJsonObject + + for(field in fieldNames){ + val valueBefore = objBefore.get(field) + val valueAfter = objAfter.get(field) + + if(valueBefore != valueAfter){ + return true + } + } + + return false + } catch (e: Exception) { + // JSON parsing failed, fallback to full comparison + return bodyBefore != bodyAfter + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index 762bf3759d..af38757328 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -6,6 +6,7 @@ import org.evomaster.core.problem.rest.* import org.evomaster.core.problem.rest.builder.RestIndividualSelectorUtils import org.evomaster.core.problem.rest.data.HttpVerb import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.rest.service.fitness.RestFitness import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler @@ -45,6 +46,9 @@ class HttpSemanticsService { @Inject private lateinit var idMapper: IdMapper + @Inject + private lateinit var builder: RestIndividualBuilder + /** * All actions that can be defined from the OpenAPI schema */ @@ -90,6 +94,8 @@ class HttpSemanticsService { // – A repeated followup PUT with 201 on same endpoint should not return 201 (must enforce 200 or 204) putRepeatedCreated() + + sideEffectsOfFailedModification() } /** @@ -198,4 +204,97 @@ class HttpSemanticsService { } } + /** + * Checking bugs like: + * POST|PUT /X 2xx (create resource) + * GET /X 2xx (before state) + * PUT|PATCH /X 4xx (failed modification) + * GET /X 2xx (after state - should be same as before) + * + * If a PUT/PATCH fails with 4xx, it should have no side-effects. + * A GET before and after should return the same resource state. + */ + private fun sideEffectsOfFailedModification() { + + val verbs = listOf(HttpVerb.PUT, HttpVerb.PATCH) + + for (verb in verbs) { + + val modifyOperations = RestIndividualSelectorUtils.getAllActionDefinitions(actionDefinitions, verb) + + modifyOperations.forEach { modOp -> + + // check that a GET definition exists for this same path + val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == modOp.path } + ?: return@forEach + + // find individuals that have a 4xx PUT/PATCH on this path + val failedModifyIndividuals = RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, + verb, + modOp.path, + statusGroup = StatusGroup.G_4xx + ) + if (failedModifyIndividuals.isEmpty()) { + return@forEach + } + + // among those, find one that also has a successful creation step + // (POST 2xx on parent path, or PUT 201 on same path) + val parentPath = if (!modOp.path.isRoot()) modOp.path.parentPath() else null + + val withCreation = failedModifyIndividuals.filter { ind -> + ind.evaluatedMainActions().any { ea -> + val action = ea.action as RestCallAction + val result = ea.result as RestCallResult + (parentPath != null + && action.verb == HttpVerb.POST + && action.path.isEquivalent(parentPath) + && StatusGroup.G_2xx.isInGroup(result.getStatusCode())) + || + (action.verb == HttpVerb.PUT + && action.path.isEquivalent(modOp.path) + && result.getStatusCode() == 201) + } + } + if (withCreation.isEmpty()) { + return@forEach + } + + val selected = withCreation.minBy { it.individual.size() } + + // slice up to the 4xx PUT/PATCH + val failedIndex = RestIndividualSelectorUtils.findIndexOfAction( + selected, verb, modOp.path, statusGroup = StatusGroup.G_4xx + ) + if (failedIndex < 0) { + return@forEach + } + + val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction( + selected.individual, failedIndex + ) + + val actions = ind.seeMainExecutableActions() + val last = actions[actions.size - 1] // the 4xx PUT/PATCH + + // add GET before the failed PUT/PATCH + val getBefore = getDef.copy() as RestCallAction + getBefore.doInitialize(randomness) + getBefore.forceNewTaints() + getBefore.bindToSamePathResolution(last) + getBefore.auth = last.auth + ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) + + // add GET after the failed PUT/PATCH + val getAfter = getBefore.copy() as RestCallAction + getAfter.resetLocalIdRecursively() + ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) + + prepareEvaluateAndSave(ind) + } + } + } + + } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index f90e48d33f..0455e35ba6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1217,6 +1217,36 @@ abstract class AbstractRestFitness : HttpWsFitness() { } else { handleRepeatedCreatePut(individual, actionResults, fv) } + + if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION)) { + LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration") + } else { + handleFailedModification(individual, actionResults, fv) + } + } + + private fun handleFailedModification( + individual: RestIndividual, + actionResults: List, + fv: FitnessValue + ) { + val issues = HttpSemanticsOracle.hasSideEffectFailedModification(individual,actionResults) + if(!issues){ + return + } + + val putOrPatch = individual.seeMainExecutableActions().filter { + it.verb == HttpVerb.PUT || it.verb == HttpVerb.PATCH + }.last() + + val category = ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, putOrPatch.getName()) + ) + fv.updateTarget(scenarioId, 1.0, individual.seeMainExecutableActions().lastIndex) + + val ar = actionResults.find { it.sourceLocalId == putOrPatch.getLocalId() } as RestCallResult? + ?: return + ar.addFault(DetectedFault(category, putOrPatch.getName(), null)) } private fun handleRepeatedCreatePut( diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt new file mode 100644 index 0000000000..9919464bb7 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt @@ -0,0 +1,116 @@ +package org.evomaster.core.problem.rest.oracle + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class HttpSemanticsOracleTest { + + @Test + fun testUnchangedModifiedFieldReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","ts":"2026-01-01"}""", + bodyAfter = """{"name":"Doe","ts":"2026-01-02"}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testNoModifiedFieldChangedReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":30}""", + bodyAfter = """{"name":"Doe","email":"a@a.com","age":31}""", + fieldNames = setOf("name", "email") + )) + } + + @Test + fun testModifiedFieldAbsentInBothBodiesReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"age":30}""", + bodyAfter = """{"age":31}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testUnchangedIntegerModifiedFieldReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":42,"label":"changed"}""", + fieldNames = setOf("count") + )) + } + + // hasChangedModifiedFields — field changed -> true + @Test + fun testChangedModifiedFieldReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","age":42}""", + bodyAfter = """{"name":"Bob","age":42}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testOneOfMultipleModifiedFieldsChangedReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":42}""", + bodyAfter = """{"name":"Doe","email":"b@b.com","age":42}""", + fieldNames = setOf("name", "email") + )) + } + + @Test + fun testModifiedFieldPresentInBeforeButAbsentInAfterReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe"}""", + bodyAfter = """{"age":42}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testChangedIntegerModifiedFieldReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":44,"label":"test"}""", + fieldNames = setOf("count") + )) + } + + @Test + fun testInvalidJsonDifferentBodiesFallbackReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "also not valid json", + fieldNames = setOf("name") + )) + } + + @Test + fun testInvalidJsonSameBodiesFallbackReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "not valid json", + fieldNames = setOf("name") + )) + } + + @Test + fun testJsonArrayDifferentBodiesFallbackReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """[{"name":"Doe"}]""", + bodyAfter = """[{"name":"Bob"}]""", + fieldNames = setOf("name") + )) + } + + @Test + fun testJsonArraySameBodiesFallbackReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """[{"name":"Doe"}]""", + bodyAfter = """[{"name":"Doe"}]""", + fieldNames = setOf("name") + )) + } +}