From 3f4d0718ac1a674ded59fab6b6c2e87b195fcdbc Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 10:13:35 -0400 Subject: [PATCH 01/11] test runCommand when retryReads/Writes false --- .../backpressure-runCommand-requirements.yml | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 source/client-backpressure/tests/backpressure-runCommand-requirements.yml diff --git a/source/client-backpressure/tests/backpressure-runCommand-requirements.yml b/source/client-backpressure/tests/backpressure-runCommand-requirements.yml new file mode 100644 index 0000000000..599bcef85c --- /dev/null +++ b/source/client-backpressure/tests/backpressure-runCommand-requirements.yml @@ -0,0 +1,141 @@ +description: tests that runCommand only retries when retryReads=true and retryWrites=true + +schemaVersion: '1.3' + +runOnRequirements: + - minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - client: + id: &client_defaults client_defaults + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - database: + id: &database_defaults database_defaults + client: *client_defaults + databaseName: backpressure-runCommand-requirements-db + + - client: + id: &client_retryReads_false client_retryReads_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + uriOptions: + retryReads: false + + - database: + id: &database_retryReads_false database_retryReads_false + client: *client_retryReads_false + databaseName: backpressure-runCommand-requirements-db + + - client: + id: &client_retryWrites_false client_retryWrites_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + uriOptions: + retryWrites: false + + - database: + id: &database_retryWrites_false database_retryWrites_false + client: *client_retryWrites_false + databaseName: backpressure-runCommand-requirements-db + + - client: + id: &internal_client internal_client + useMultipleMongoses: false + +tests: + - description: 'database.runCommand retries with defaults (retryReads=true and retryWrites=true)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database_defaults + arguments: + command: { ping: 1 } + commandName: ping + + expectEvents: + - client: *client_defaults + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - description: 'database.runCommand does not retry when retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database_retryReads_false + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + - description: 'database.runCommand does not retry when retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database_retryWrites_false + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping From fe86596a7f09d69836bc93ac8592a40e735bf39f Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 10:13:48 -0400 Subject: [PATCH 02/11] generate JSON files --- .../backpressure-runCommand-requirements.json | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 source/client-backpressure/tests/backpressure-runCommand-requirements.json diff --git a/source/client-backpressure/tests/backpressure-runCommand-requirements.json b/source/client-backpressure/tests/backpressure-runCommand-requirements.json new file mode 100644 index 0000000000..38d2dc56c5 --- /dev/null +++ b/source/client-backpressure/tests/backpressure-runCommand-requirements.json @@ -0,0 +1,267 @@ +{ + "description": "tests that runCommand only retries when retryReads=true and retryWrites=true", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client_defaults", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database_defaults", + "client": "client_defaults", + "databaseName": "backpressure-runCommand-requirements-db" + } + }, + { + "client": { + "id": "client_retryReads_false", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "uriOptions": { + "retryReads": false + } + } + }, + { + "database": { + "id": "database_retryReads_false", + "client": "client_retryReads_false", + "databaseName": "backpressure-runCommand-requirements-db" + } + }, + { + "client": { + "id": "client_retryWrites_false", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "uriOptions": { + "retryWrites": false + } + } + }, + { + "database": { + "id": "database_retryWrites_false", + "client": "client_retryWrites_false", + "databaseName": "backpressure-runCommand-requirements-db" + } + }, + { + "client": { + "id": "internal_client", + "useMultipleMongoses": false + } + } + ], + "tests": [ + { + "description": "database.runCommand retries with defaults (retryReads=true and retryWrites=true)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database_defaults", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client_defaults", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.runCommand does not retry when retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database_retryReads_false", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.runCommand does not retry when retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database_retryWrites_false", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + } + ] +} From 38345b4a4559b8b81e811953d525972ef5c9257b Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 10:22:00 -0400 Subject: [PATCH 03/11] fix format Drive-by fix. Ran: `pre-commit run --all-files mdformat`. --- source/retryable-reads/retryable-reads.md | 1 + source/retryable-writes/retryable-writes.md | 1 + 2 files changed, 2 insertions(+) diff --git a/source/retryable-reads/retryable-reads.md b/source/retryable-reads/retryable-reads.md index 07e1ed6ef0..61f7ed34ce 100644 --- a/source/retryable-reads/retryable-reads.md +++ b/source/retryable-reads/retryable-reads.md @@ -579,6 +579,7 @@ reads because the resilience benefit of retryable reads outweighs the minor risk any customers experiencing degraded performance can simply disable `retryableReads`. ## Changelog + - 2026-02-19: Clarified that server deprioritization on replica sets only occurs for `SystemOverloadedError` errors. - 2026-02-11: Clarified that the retry logic and pseudocode does not include the modifications required by client diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index fb89f119f8..85b2ec4cdb 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -711,6 +711,7 @@ which only happens when the retryWrites option is true on the client. For the dr retryWrites is not true would be inconsistent with the server and potentially confusing to developers. ## Changelog + - 2026-02-19: Clarified that server deprioritization on replica sets only occurs for `SystemOverloadedError` errors. - 2026-02-11: Clarified that the retry logic and pseudocode does not include the modifications required by client From 857064ebce7b136ce1fb3f6875bb14ccda465371 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 14:27:53 -0400 Subject: [PATCH 04/11] remove unnecessary `sys` --- source/etc/generate-backpressure-retryability-tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/etc/generate-backpressure-retryability-tests.py b/source/etc/generate-backpressure-retryability-tests.py index 3d8d914e1f..e8734d6114 100644 --- a/source/etc/generate-backpressure-retryability-tests.py +++ b/source/etc/generate-backpressure-retryability-tests.py @@ -1,7 +1,6 @@ from collections import namedtuple from jinja2 import Template import os -import sys Operation = namedtuple( 'Operation', ['operation_name', 'command_name', 'object', 'arguments']) From 165d5ad924a3792bfa1d8b522a87978a604f8f4f Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 14:30:19 -0400 Subject: [PATCH 05/11] add `operation_type` --- ...enerate-backpressure-retryability-tests.py | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/source/etc/generate-backpressure-retryability-tests.py b/source/etc/generate-backpressure-retryability-tests.py index e8734d6114..2a14b1bf98 100644 --- a/source/etc/generate-backpressure-retryability-tests.py +++ b/source/etc/generate-backpressure-retryability-tests.py @@ -3,7 +3,7 @@ import os Operation = namedtuple( - 'Operation', ['operation_name', 'command_name', 'object', 'arguments']) + 'Operation', ['operation_name', 'command_name', 'object', 'arguments', 'operation_type']) CLIENT_BULK_WRITE_ARGUMENTS = '''models: - insertOne: @@ -11,10 +11,10 @@ document: { _id: 8, x: 88 }''' CLIENT_OPERATIONS = [ - Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']), - Operation('listDatabaseNames', 'listDatabases', 'client', []), - Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []']), - Operation('clientBulkWrite', 'bulkWrite', 'client', [CLIENT_BULK_WRITE_ARGUMENTS]) + Operation('listDatabases', 'listDatabases', 'client', ['filter: {}'], 'read'), + Operation('listDatabaseNames', 'listDatabases', 'client', [], 'read'), + Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []'], 'read'), + Operation('clientBulkWrite', 'bulkWrite', 'client', [CLIENT_BULK_WRITE_ARGUMENTS], 'write') ] RUN_COMMAND_ARGUMENTS = '''command: { ping: 1 } @@ -22,13 +22,13 @@ DB_OPERATIONS = [ Operation('aggregate', 'aggregate', 'database', [ - 'pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]']), + 'pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]'], 'read'), Operation('listCollections', 'listCollections', - 'database', ['filter: {}']), + 'database', ['filter: {}'], 'read'), Operation('listCollectionNames', 'listCollections', - 'database', ['filter: {}']), # Optional. - Operation('runCommand', 'ping', 'database', [RUN_COMMAND_ARGUMENTS]), - Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []']) + 'database', ['filter: {}'], 'read'), # Optional. + Operation('runCommand', 'ping', 'database', [RUN_COMMAND_ARGUMENTS], 'read'), + Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []'], 'read') ] INSERT_MANY_ARGUMENTS = '''documents: @@ -39,43 +39,43 @@ document: { _id: 2, x: 22 }''' COLLECTION_READ_OPERATIONS = [ - Operation('aggregate', 'aggregate', 'collection', ['pipeline: []']), - # Operation('count', 'count', 'collection', ['filter: {}']), # Deprecated. - Operation('countDocuments', 'aggregate', 'collection', ['filter: {}']), - Operation('estimatedDocumentCount', 'count', 'collection', []), + Operation('aggregate', 'aggregate', 'collection', ['pipeline: []'], 'read'), + # Operation('count', 'count', 'collection', ['filter: {}'], 'read'), # Deprecated. + Operation('countDocuments', 'aggregate', 'collection', ['filter: {}'], 'read'), + Operation('estimatedDocumentCount', 'count', 'collection', [], 'read'), Operation('distinct', 'distinct', 'collection', - ['fieldName: x', 'filter: {}']), - Operation('find', 'find', 'collection', ['filter: {}']), - Operation('findOne', 'find', 'collection', ['filter: {}']), # Optional. - Operation('listIndexes', 'listIndexes', 'collection', []), - Operation('listIndexNames', 'listIndexes', 'collection', []), # Optional. + ['fieldName: x', 'filter: {}'], 'read'), + Operation('find', 'find', 'collection', ['filter: {}'], 'read'), + Operation('findOne', 'find', 'collection', ['filter: {}'], 'read'), # Optional. + Operation('listIndexes', 'listIndexes', 'collection', [], 'read'), + Operation('listIndexNames', 'listIndexes', 'collection', [], 'read'), # Optional. Operation('createChangeStream', 'aggregate', - 'collection', ['pipeline: []']), + 'collection', ['pipeline: []'], 'read'), ] COLLECTION_WRITE_OPERATIONS = [ Operation('insertOne', 'insert', 'collection', - ['document: { _id: 2, x: 22 }']), - Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS]), - Operation('deleteOne', 'delete', 'collection', ['filter: {}']), - Operation('deleteMany', 'delete', 'collection', ['filter: {}']), + ['document: { _id: 2, x: 22 }'], 'write'), + Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS], 'write'), + Operation('deleteOne', 'delete', 'collection', ['filter: {}'], 'write'), + Operation('deleteMany', 'delete', 'collection', ['filter: {}'], 'write'), Operation('replaceOne', 'update', 'collection', [ - 'filter: {}', 'replacement: { x: 22 }']), + 'filter: {}', 'replacement: { x: 22 }'], 'write'), Operation('updateOne', 'update', 'collection', [ - 'filter: {}', 'update: { $set: { x: 22 } }']), + 'filter: {}', 'update: { $set: { x: 22 } }'], 'write'), Operation('updateMany', 'update', 'collection', [ - 'filter: {}', 'update: { $set: { x: 22 } }']), + 'filter: {}', 'update: { $set: { x: 22 } }'], 'write'), Operation('findOneAndDelete', 'findAndModify', - 'collection', ['filter: {}']), + 'collection', ['filter: {}'], 'write'), Operation('findOneAndReplace', 'findAndModify', 'collection', - ['filter: {}', 'replacement: { x: 22 }']), + ['filter: {}', 'replacement: { x: 22 }'], 'write'), Operation('findOneAndUpdate', 'findAndModify', 'collection', - ['filter: {}', 'update: { $set: { x: 22 } }']), - Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS]), + ['filter: {}', 'update: { $set: { x: 22 } }'], 'write'), + Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS], 'write'), Operation('createIndex', 'createIndexes', 'collection', - ['keys: { x: 11 }', 'name: "x_11"']), - Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"']), - Operation('dropIndexes', 'dropIndexes', 'collection', []), + ['keys: { x: 11 }', 'name: "x_11"'], 'write'), + Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"'], 'write'), + Operation('dropIndexes', 'dropIndexes', 'collection', [], 'write'), ] COLLECTION_OPERATIONS = COLLECTION_READ_OPERATIONS + COLLECTION_WRITE_OPERATIONS @@ -121,4 +121,4 @@ def generate_max_attempts_tests(): tests_dir, OPERATIONS) generate_retry_loop_tests() -generate_max_attempts_tests() \ No newline at end of file +generate_max_attempts_tests() From ce8fb140c50f6d777ce14fefc29cde0426bc7d61 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 14:57:51 -0400 Subject: [PATCH 06/11] test read operations do not retry if retryReads=false --- .../tests/backpressure-retry-loop.json | 1345 +++++++++++++++-- .../tests/backpressure-retry-loop.yml | 555 +++++++ .../backpressure-retry-loop.yml.template | 55 + 3 files changed, 1789 insertions(+), 166 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 6e64bfe619..db1c4b5d27 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -59,6 +59,68 @@ "database": "database", "collectionName": "coll" } + }, + { + "client": { + "id": "client_retryReads_false", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "ignoreCommandMonitoringEvents": [ + "killCursors" + ], + "uriOptions": { + "retryReads": false + } + } + }, + { + "database": { + "id": "database_retryReads_false", + "client": "client_retryReads_false", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection_retryReads_false", + "database": "database_retryReads_false", + "collectionName": "coll" + } + }, + { + "client": { + "id": "client_retryWrites_false", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "ignoreCommandMonitoringEvents": [ + "killCursors" + ], + "uriOptions": { + "retryWrites": false + } + } + }, + { + "database": { + "id": "database_retryWrites_false", + "client": "client_retryWrites_false", + "databaseName": "backpressure-db" + } + }, + { + "collection": { + "id": "collection_retryWrites_false", + "database": "database_retryWrites_false", + "collectionName": "coll" + } } ], "initialData": [ @@ -154,6 +216,62 @@ } ] }, + { + "description": "client.listDatabases (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabases", + "object": "client_retryReads_false", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, { "description": "client.listDatabaseNames retries using operation loop", "operations": [ @@ -233,6 +351,59 @@ } ] }, + { + "description": "client.listDatabaseNames (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabaseNames", + "object": "client_retryReads_false", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, { "description": "client.createChangeStream retries using operation loop", "operations": [ @@ -315,6 +486,62 @@ } ] }, + { + "description": "client.createChangeStream (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "client_retryReads_false", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, { "description": "client.clientBulkWrite retries using operation loop", "runOnRequirements": [ @@ -502,7 +729,7 @@ ] }, { - "description": "database.listCollections retries using operation loop", + "description": "database.aggregate (read) does not retry if retryReads=false", "operations": [ { "name": "failPoint", @@ -512,11 +739,11 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ - "listCollections" + "aggregate" ], "errorLabels": [ "RetryableError", @@ -528,55 +755,36 @@ } }, { - "name": "listCollections", - "object": "database", + "name": "aggregate", + "object": "database_retryReads_false", "arguments": { - "filter": {} + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryReads_false", "events": [ { "commandStartedEvent": { - "commandName": "listCollections" - } - }, - { - "commandFailedEvent": { - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "commandName": "listCollections" - } - }, - { - "commandFailedEvent": { - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "commandName": "listCollections" + "commandName": "aggregate" } }, { "commandFailedEvent": { - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "commandName": "listCollections" - } - }, - { - "commandSucceededEvent": { - "commandName": "listCollections" + "commandName": "aggregate" } } ] @@ -584,7 +792,7 @@ ] }, { - "description": "database.listCollectionNames retries using operation loop", + "description": "database.listCollections retries using operation loop", "operations": [ { "name": "failPoint", @@ -610,7 +818,7 @@ } }, { - "name": "listCollectionNames", + "name": "listCollections", "object": "database", "arguments": { "filter": {} @@ -666,7 +874,7 @@ ] }, { - "description": "database.runCommand retries using operation loop", + "description": "database.listCollections (read) does not retry if retryReads=false", "operations": [ { "name": "failPoint", @@ -676,11 +884,11 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ - "ping" + "listCollections" ], "errorLabels": [ "RetryableError", @@ -692,13 +900,66 @@ } }, { - "name": "runCommand", - "object": "database", + "name": "listCollections", + "object": "database_retryReads_false", "arguments": { - "command": { - "ping": 1 - }, - "commandName": "ping" + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollectionNames", + "object": "database", + "arguments": { + "filter": {} } } ], @@ -708,42 +969,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandFailedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandStartedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandFailedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandStartedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandFailedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandStartedEvent": { - "commandName": "ping" + "commandName": "listCollections" } }, { "commandSucceededEvent": { - "commandName": "ping" + "commandName": "listCollections" } } ] @@ -751,7 +1012,63 @@ ] }, { - "description": "database.createChangeStream retries using operation loop", + "description": "database.listCollectionNames (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollectionNames", + "object": "database_retryReads_false", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", "operations": [ { "name": "failPoint", @@ -765,7 +1082,7 @@ }, "data": { "failCommands": [ - "aggregate" + "ping" ], "errorLabels": [ "RetryableError", @@ -777,10 +1094,13 @@ } }, { - "name": "createChangeStream", + "name": "runCommand", "object": "database", "arguments": { - "pipeline": [] + "command": { + "ping": 1 + }, + "commandName": "ping" } } ], @@ -790,42 +1110,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandFailedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandStartedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandFailedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandStartedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandFailedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandStartedEvent": { - "commandName": "aggregate" + "commandName": "ping" } }, { "commandSucceededEvent": { - "commandName": "aggregate" + "commandName": "ping" } } ] @@ -833,7 +1153,66 @@ ] }, { - "description": "collection.aggregate retries using operation loop", + "description": "database.runCommand (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database_retryReads_false", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", "operations": [ { "name": "failPoint", @@ -859,8 +1238,8 @@ } }, { - "name": "aggregate", - "object": "collection", + "name": "createChangeStream", + "object": "database", "arguments": { "pipeline": [] } @@ -915,7 +1294,63 @@ ] }, { - "description": "collection.countDocuments retries using operation loop", + "description": "database.createChangeStream (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "database_retryReads_false", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", "operations": [ { "name": "failPoint", @@ -941,10 +1376,10 @@ } }, { - "name": "countDocuments", + "name": "aggregate", "object": "collection", "arguments": { - "filter": {} + "pipeline": [] } } ], @@ -984,12 +1419,478 @@ }, { "commandStartedEvent": { - "commandName": "aggregate" + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection_retryReads_false", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "countDocuments", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "countDocuments", + "object": "collection_retryReads_false", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "estimatedDocumentCount", + "object": "collection" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "estimatedDocumentCount", + "object": "collection_retryReads_false", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "distinct", + "object": "collection", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.distinct (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "distinct", + "object": "collection_retryReads_false", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" } }, { - "commandSucceededEvent": { - "commandName": "aggregate" + "commandFailedEvent": { + "commandName": "distinct" } } ] @@ -997,7 +1898,7 @@ ] }, { - "description": "collection.estimatedDocumentCount retries using operation loop", + "description": "collection.find retries using operation loop", "operations": [ { "name": "failPoint", @@ -1011,7 +1912,7 @@ }, "data": { "failCommands": [ - "count" + "find" ], "errorLabels": [ "RetryableError", @@ -1023,8 +1924,11 @@ } }, { - "name": "estimatedDocumentCount", - "object": "collection" + "name": "find", + "object": "collection", + "arguments": { + "filter": {} + } } ], "expectEvents": [ @@ -1033,42 +1937,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandFailedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandStartedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandFailedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandStartedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandFailedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandStartedEvent": { - "commandName": "count" + "commandName": "find" } }, { "commandSucceededEvent": { - "commandName": "count" + "commandName": "find" } } ] @@ -1076,7 +1980,7 @@ ] }, { - "description": "collection.distinct retries using operation loop", + "description": "collection.find (read) does not retry if retryReads=false", "operations": [ { "name": "failPoint", @@ -1086,11 +1990,11 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ - "distinct" + "find" ], "errorLabels": [ "RetryableError", @@ -1102,56 +2006,29 @@ } }, { - "name": "distinct", - "object": "collection", + "name": "find", + "object": "collection_retryReads_false", "arguments": { - "fieldName": "x", "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryReads_false", "events": [ { "commandStartedEvent": { - "commandName": "distinct" - } - }, - { - "commandFailedEvent": { - "commandName": "distinct" - } - }, - { - "commandStartedEvent": { - "commandName": "distinct" - } - }, - { - "commandFailedEvent": { - "commandName": "distinct" - } - }, - { - "commandStartedEvent": { - "commandName": "distinct" + "commandName": "find" } }, { "commandFailedEvent": { - "commandName": "distinct" - } - }, - { - "commandStartedEvent": { - "commandName": "distinct" - } - }, - { - "commandSucceededEvent": { - "commandName": "distinct" + "commandName": "find" } } ] @@ -1159,7 +2036,7 @@ ] }, { - "description": "collection.find retries using operation loop", + "description": "collection.findOne retries using operation loop", "operations": [ { "name": "failPoint", @@ -1185,7 +2062,7 @@ } }, { - "name": "find", + "name": "findOne", "object": "collection", "arguments": { "filter": {} @@ -1241,7 +2118,7 @@ ] }, { - "description": "collection.findOne retries using operation loop", + "description": "collection.findOne (read) does not retry if retryReads=false", "operations": [ { "name": "failPoint", @@ -1251,7 +2128,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ @@ -1268,15 +2145,19 @@ }, { "name": "findOne", - "object": "collection", + "object": "collection_retryReads_false", "arguments": { "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryReads_false", "events": [ { "commandStartedEvent": { @@ -1287,36 +2168,6 @@ "commandFailedEvent": { "commandName": "find" } - }, - { - "commandStartedEvent": { - "commandName": "find" - } - }, - { - "commandFailedEvent": { - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "commandName": "find" - } - }, - { - "commandFailedEvent": { - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "commandName": "find" - } - }, - { - "commandSucceededEvent": { - "commandName": "find" - } } ] } @@ -1401,6 +2252,59 @@ } ] }, + { + "description": "collection.listIndexes (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexes", + "object": "collection_retryReads_false", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, { "description": "collection.listIndexNames retries using operation loop", "operations": [ @@ -1480,6 +2384,59 @@ } ] }, + { + "description": "collection.listIndexNames (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexNames", + "object": "collection_retryReads_false", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, { "description": "collection.createChangeStream retries using operation loop", "operations": [ @@ -1562,6 +2519,62 @@ } ] }, + { + "description": "collection.createChangeStream (read) does not retry if retryReads=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "collection_retryReads_false", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryReads_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, { "description": "collection.insertOne retries using operation loop", "operations": [ diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index cbe43b416d..aefd2ffc4c 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -39,6 +39,42 @@ createEntities: database: *database collectionName: *collection_name + - client: + id: &client_retryReads_false client_retryReads_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + ignoreCommandMonitoringEvents: [killCursors] + uriOptions: + retryReads: false + + - database: + id: &database_retryReads_false database_retryReads_false + client: *client_retryReads_false + databaseName: *database_name + + - collection: + id: &collection_retryReads_false collection_retryReads_false + database: *database_retryReads_false + collectionName: *collection_name + + - client: + id: &client_retryWrites_false client_retryWrites_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + ignoreCommandMonitoringEvents: [killCursors] + uriOptions: + retryWrites: false + + - database: + id: &database_retryWrites_false database_retryWrites_false + client: *client_retryWrites_false + databaseName: backpressure-db + + - collection: + id: &collection_retryWrites_false collection_retryWrites_false + database: *database_retryWrites_false + collectionName: *collection_name + initialData: - collectionName: *collection_name databaseName: *database_name @@ -86,6 +122,36 @@ tests: commandName: listDatabases - commandSucceededEvent: commandName: listDatabases + - description: 'client.listDatabases (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabases + object: *client_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - description: 'client.listDatabaseNames retries using operation loop' operations: @@ -123,6 +189,34 @@ tests: commandName: listDatabases - commandSucceededEvent: commandName: listDatabases + - description: 'client.listDatabaseNames (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabaseNames + object: *client_retryReads_false + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - description: 'client.createChangeStream retries using operation loop' operations: @@ -162,6 +256,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'client.createChangeStream (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *client_retryReads_false + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'client.clientBulkWrite retries using operation loop' runOnRequirements: @@ -207,6 +331,7 @@ tests: - commandSucceededEvent: commandName: bulkWrite + - description: 'database.aggregate retries using operation loop' operations: - name: failPoint @@ -245,6 +370,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'database.aggregate (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *database_retryReads_false + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'database.listCollections retries using operation loop' operations: @@ -284,6 +439,36 @@ tests: commandName: listCollections - commandSucceededEvent: commandName: listCollections + - description: 'database.listCollections (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollections + object: *database_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - description: 'database.listCollectionNames retries using operation loop' operations: @@ -323,6 +508,36 @@ tests: commandName: listCollections - commandSucceededEvent: commandName: listCollections + - description: 'database.listCollectionNames (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollectionNames + object: *database_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - description: 'database.runCommand retries using operation loop' operations: @@ -363,6 +578,37 @@ tests: commandName: ping - commandSucceededEvent: commandName: ping + - description: 'database.runCommand (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database_retryReads_false + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - description: 'database.createChangeStream retries using operation loop' operations: @@ -402,6 +648,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'database.createChangeStream (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *database_retryReads_false + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'collection.aggregate retries using operation loop' operations: @@ -441,6 +717,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'collection.aggregate (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection_retryReads_false + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'collection.countDocuments retries using operation loop' operations: @@ -480,6 +786,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'collection.countDocuments (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: countDocuments + object: *collection_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'collection.estimatedDocumentCount retries using operation loop' operations: @@ -517,6 +853,34 @@ tests: commandName: count - commandSucceededEvent: commandName: count + - description: 'collection.estimatedDocumentCount (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: estimatedDocumentCount + object: *collection_retryReads_false + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - description: 'collection.distinct retries using operation loop' operations: @@ -557,6 +921,37 @@ tests: commandName: distinct - commandSucceededEvent: commandName: distinct + - description: 'collection.distinct (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: distinct + object: *collection_retryReads_false + arguments: + fieldName: x + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - description: 'collection.find retries using operation loop' operations: @@ -596,6 +991,36 @@ tests: commandName: find - commandSucceededEvent: commandName: find + - description: 'collection.find (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + object: *collection_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - description: 'collection.findOne retries using operation loop' operations: @@ -635,6 +1060,36 @@ tests: commandName: find - commandSucceededEvent: commandName: find + - description: 'collection.findOne (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOne + object: *collection_retryReads_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - description: 'collection.listIndexes retries using operation loop' operations: @@ -672,6 +1127,34 @@ tests: commandName: listIndexes - commandSucceededEvent: commandName: listIndexes + - description: 'collection.listIndexes (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexes + object: *collection_retryReads_false + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - description: 'collection.listIndexNames retries using operation loop' operations: @@ -709,6 +1192,34 @@ tests: commandName: listIndexes - commandSucceededEvent: commandName: listIndexes + - description: 'collection.listIndexNames (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexNames + object: *collection_retryReads_false + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - description: 'collection.createChangeStream retries using operation loop' operations: @@ -748,6 +1259,36 @@ tests: commandName: aggregate - commandSucceededEvent: commandName: aggregate + - description: 'collection.createChangeStream (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *collection_retryReads_false + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - description: 'collection.insertOne retries using operation loop' operations: @@ -788,6 +1329,7 @@ tests: - commandSucceededEvent: commandName: insert + - description: 'collection.insertMany retries using operation loop' operations: - name: failPoint @@ -828,6 +1370,7 @@ tests: - commandSucceededEvent: commandName: insert + - description: 'collection.deleteOne retries using operation loop' operations: - name: failPoint @@ -867,6 +1410,7 @@ tests: - commandSucceededEvent: commandName: delete + - description: 'collection.deleteMany retries using operation loop' operations: - name: failPoint @@ -906,6 +1450,7 @@ tests: - commandSucceededEvent: commandName: delete + - description: 'collection.replaceOne retries using operation loop' operations: - name: failPoint @@ -946,6 +1491,7 @@ tests: - commandSucceededEvent: commandName: update + - description: 'collection.updateOne retries using operation loop' operations: - name: failPoint @@ -986,6 +1532,7 @@ tests: - commandSucceededEvent: commandName: update + - description: 'collection.updateMany retries using operation loop' operations: - name: failPoint @@ -1026,6 +1573,7 @@ tests: - commandSucceededEvent: commandName: update + - description: 'collection.findOneAndDelete retries using operation loop' operations: - name: failPoint @@ -1065,6 +1613,7 @@ tests: - commandSucceededEvent: commandName: findAndModify + - description: 'collection.findOneAndReplace retries using operation loop' operations: - name: failPoint @@ -1105,6 +1654,7 @@ tests: - commandSucceededEvent: commandName: findAndModify + - description: 'collection.findOneAndUpdate retries using operation loop' operations: - name: failPoint @@ -1145,6 +1695,7 @@ tests: - commandSucceededEvent: commandName: findAndModify + - description: 'collection.bulkWrite retries using operation loop' operations: - name: failPoint @@ -1186,6 +1737,7 @@ tests: - commandSucceededEvent: commandName: insert + - description: 'collection.createIndex retries using operation loop' operations: - name: failPoint @@ -1226,6 +1778,7 @@ tests: - commandSucceededEvent: commandName: createIndexes + - description: 'collection.dropIndex retries using operation loop' operations: - name: createIndex @@ -1270,6 +1823,7 @@ tests: - commandSucceededEvent: commandName: dropIndexes + - description: 'collection.dropIndexes retries using operation loop' operations: - name: failPoint @@ -1306,3 +1860,4 @@ tests: commandName: dropIndexes - commandSucceededEvent: commandName: dropIndexes + diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index f810ff7020..a5f642bed5 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -39,6 +39,24 @@ createEntities: database: *database collectionName: *collection_name + - client: + id: &client_retryReads_false client_retryReads_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + ignoreCommandMonitoringEvents: [killCursors] + uriOptions: + retryReads: false + + - database: + id: &database_retryReads_false database_retryReads_false + client: *client_retryReads_false + databaseName: *database_name + + - collection: + id: &collection_retryReads_false collection_retryReads_false + database: *database_retryReads_false + collectionName: *collection_name + initialData: - collectionName: *collection_name databaseName: *database_name @@ -99,4 +117,41 @@ tests: {% for operation in operations %} commandName: {{operation.command_name}} - commandSucceededEvent: commandName: {{operation.command_name}} + + {%- if operation.operation_type == 'read' %} + - description: '{{operation.object}}.{{operation.operation_name}} (read) does not retry if retryReads=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: {{operation.operation_name}} + object: *{{operation.object}}_retryReads_false + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryReads_false + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + {%- endif %} + {% endfor -%} From b37cb83588884bac2fc5f7c9772a64e5a209274e Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 15:22:39 -0400 Subject: [PATCH 07/11] test write operations do not retry if retryWrites=false --- .../tests/backpressure-retry-loop.json | 1209 ++++++++++++++--- .../tests/backpressure-retry-loop.yml | 452 ++++++ .../backpressure-retry-loop.yml.template | 63 + 3 files changed, 1571 insertions(+), 153 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index db1c4b5d27..15314ab9ff 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -639,6 +639,77 @@ } ] }, + { + "description": "client.clientBulkWrite (write) does not retry if retryWrites=false", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client_retryWrites_false", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, { "description": "database.aggregate retries using operation loop", "operations": [ @@ -2660,6 +2731,65 @@ } ] }, + { + "description": "collection.insertOne (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertOne", + "object": "collection_retryWrites_false", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, { "description": "collection.insertMany retries using operation loop", "operations": [ @@ -2747,6 +2877,67 @@ } ] }, + { + "description": "collection.insertMany (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertMany", + "object": "collection_retryWrites_false", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, { "description": "collection.deleteOne retries using operation loop", "operations": [ @@ -2830,7 +3021,7 @@ ] }, { - "description": "collection.deleteMany retries using operation loop", + "description": "collection.deleteOne (write) does not retry if retryWrites=false", "operations": [ { "name": "failPoint", @@ -2840,7 +3031,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ @@ -2856,16 +3047,20 @@ } }, { - "name": "deleteMany", - "object": "collection", + "name": "deleteOne", + "object": "collection_retryWrites_false", "arguments": { "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryWrites_false", "events": [ { "commandStartedEvent": { @@ -2876,43 +3071,13 @@ "commandFailedEvent": { "commandName": "delete" } - }, - { - "commandStartedEvent": { - "commandName": "delete" - } - }, - { - "commandFailedEvent": { - "commandName": "delete" - } - }, - { - "commandStartedEvent": { - "commandName": "delete" - } - }, - { - "commandFailedEvent": { - "commandName": "delete" - } - }, - { - "commandStartedEvent": { - "commandName": "delete" - } - }, - { - "commandSucceededEvent": { - "commandName": "delete" - } } ] } ] }, { - "description": "collection.replaceOne retries using operation loop", + "description": "collection.deleteMany retries using operation loop", "operations": [ { "name": "failPoint", @@ -2926,7 +3091,7 @@ }, "data": { "failCommands": [ - "update" + "delete" ], "errorLabels": [ "RetryableError", @@ -2938,13 +3103,10 @@ } }, { - "name": "replaceOne", + "name": "deleteMany", "object": "collection", "arguments": { - "filter": {}, - "replacement": { - "x": 22 - } + "filter": {} } } ], @@ -2954,42 +3116,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandFailedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandStartedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandFailedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandStartedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandFailedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandStartedEvent": { - "commandName": "update" + "commandName": "delete" } }, { "commandSucceededEvent": { - "commandName": "update" + "commandName": "delete" } } ] @@ -2997,7 +3159,7 @@ ] }, { - "description": "collection.updateOne retries using operation loop", + "description": "collection.deleteMany (write) does not retry if retryWrites=false", "operations": [ { "name": "failPoint", @@ -3007,11 +3169,11 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ - "update" + "delete" ], "errorLabels": [ "RetryableError", @@ -3023,14 +3185,68 @@ } }, { - "name": "updateOne", + "name": "deleteMany", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "replaceOne", "object": "collection", "arguments": { "filter": {}, - "update": { - "$set": { - "x": 22 - } + "replacement": { + "x": 22 } } } @@ -3084,7 +3300,66 @@ ] }, { - "description": "collection.updateMany retries using operation loop", + "description": "collection.replaceOne (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "replaceOne", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", "operations": [ { "name": "failPoint", @@ -3110,7 +3385,7 @@ } }, { - "name": "updateMany", + "name": "updateOne", "object": "collection", "arguments": { "filter": {}, @@ -3171,7 +3446,68 @@ ] }, { - "description": "collection.findOneAndDelete retries using operation loop", + "description": "collection.updateOne (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "updateOne", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", "operations": [ { "name": "failPoint", @@ -3185,7 +3521,7 @@ }, "data": { "failCommands": [ - "findAndModify" + "update" ], "errorLabels": [ "RetryableError", @@ -3197,10 +3533,15 @@ } }, { - "name": "findOneAndDelete", + "name": "updateMany", "object": "collection", "arguments": { - "filter": {} + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } } } ], @@ -3210,42 +3551,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandSucceededEvent": { - "commandName": "findAndModify" + "commandName": "update" } } ] @@ -3253,7 +3594,7 @@ ] }, { - "description": "collection.findOneAndReplace retries using operation loop", + "description": "collection.updateMany (write) does not retry if retryWrites=false", "operations": [ { "name": "failPoint", @@ -3263,11 +3604,11 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ - "findAndModify" + "update" ], "errorLabels": [ "RetryableError", @@ -3279,40 +3620,456 @@ } }, { - "name": "findOneAndReplace", - "object": "collection", + "name": "updateMany", + "object": "collection_retryWrites_false", "arguments": { "filter": {}, - "replacement": { - "x": 22 + "update": { + "$set": { + "x": 22 + } } + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryWrites_false", "events": [ { "commandStartedEvent": { - "commandName": "findAndModify" - } - }, - { - "commandFailedEvent": { - "commandName": "findAndModify" - } - }, - { - "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "update" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "update" } - }, + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndDelete", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndDelete", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndReplace", + "object": "collection", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndReplace", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection_retryWrites_false", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ { "commandStartedEvent": { "commandName": "findAndModify" @@ -3322,23 +4079,13 @@ "commandFailedEvent": { "commandName": "findAndModify" } - }, - { - "commandStartedEvent": { - "commandName": "findAndModify" - } - }, - { - "commandSucceededEvent": { - "commandName": "findAndModify" - } } ] } ] }, { - "description": "collection.findOneAndUpdate retries using operation loop", + "description": "collection.bulkWrite retries using operation loop", "operations": [ { "name": "failPoint", @@ -3352,7 +4099,7 @@ }, "data": { "failCommands": [ - "findAndModify" + "insert" ], "errorLabels": [ "RetryableError", @@ -3364,15 +4111,19 @@ } }, { - "name": "findOneAndUpdate", + "name": "bulkWrite", "object": "collection", "arguments": { - "filter": {}, - "update": { - "$set": { - "x": 22 + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } } - } + ] } } ], @@ -3382,42 +4133,42 @@ "events": [ { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandFailedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandStartedEvent": { - "commandName": "findAndModify" + "commandName": "insert" } }, { "commandSucceededEvent": { - "commandName": "findAndModify" + "commandName": "insert" } } ] @@ -3425,7 +4176,7 @@ ] }, { - "description": "collection.bulkWrite retries using operation loop", + "description": "collection.bulkWrite (write) does not retry if retryWrites=false", "operations": [ { "name": "failPoint", @@ -3435,7 +4186,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 3 + "times": 1 }, "data": { "failCommands": [ @@ -3452,7 +4203,7 @@ }, { "name": "bulkWrite", - "object": "collection", + "object": "collection_retryWrites_false", "arguments": { "requests": [ { @@ -3464,12 +4215,16 @@ } } ] + }, + "expectError": { + "isError": true, + "isClientError": false } } ], "expectEvents": [ { - "client": "client", + "client": "client_retryWrites_false", "events": [ { "commandStartedEvent": { @@ -3480,36 +4235,6 @@ "commandFailedEvent": { "commandName": "insert" } - }, - { - "commandStartedEvent": { - "commandName": "insert" - } - }, - { - "commandFailedEvent": { - "commandName": "insert" - } - }, - { - "commandStartedEvent": { - "commandName": "insert" - } - }, - { - "commandFailedEvent": { - "commandName": "insert" - } - }, - { - "commandStartedEvent": { - "commandName": "insert" - } - }, - { - "commandSucceededEvent": { - "commandName": "insert" - } } ] } @@ -3600,6 +4325,65 @@ } ] }, + { + "description": "collection.createIndex (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createIndex", + "object": "collection_retryWrites_false", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, { "description": "collection.dropIndex retries using operation loop", "operations": [ @@ -3692,6 +4476,72 @@ } ] }, + { + "description": "collection.dropIndex (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "createIndex", + "object": "retryable-writes-tests", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndex", + "object": "collection_retryWrites_false", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, { "description": "collection.dropIndexes retries using operation loop", "operations": [ @@ -3770,6 +4620,59 @@ ] } ] + }, + { + "description": "collection.dropIndexes (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndexes", + "object": "collection_retryWrites_false", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] } ] } diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index aefd2ffc4c..dfc026a79d 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -330,6 +330,40 @@ tests: commandName: bulkWrite - commandSucceededEvent: commandName: bulkWrite + - description: 'client.clientBulkWrite (write) does not retry if retryWrites=false' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: clientBulkWrite + object: *client_retryWrites_false + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite - description: 'database.aggregate retries using operation loop' @@ -1328,6 +1362,35 @@ tests: commandName: insert - commandSucceededEvent: commandName: insert + - description: 'collection.insertOne (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertOne + object: *collection_retryWrites_false + arguments: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert - description: 'collection.insertMany retries using operation loop' @@ -1369,6 +1432,36 @@ tests: commandName: insert - commandSucceededEvent: commandName: insert + - description: 'collection.insertMany (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertMany + object: *collection_retryWrites_false + arguments: + documents: + - { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert - description: 'collection.deleteOne retries using operation loop' @@ -1409,6 +1502,35 @@ tests: commandName: delete - commandSucceededEvent: commandName: delete + - description: 'collection.deleteOne (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteOne + object: *collection_retryWrites_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete - description: 'collection.deleteMany retries using operation loop' @@ -1449,6 +1571,35 @@ tests: commandName: delete - commandSucceededEvent: commandName: delete + - description: 'collection.deleteMany (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteMany + object: *collection_retryWrites_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete - description: 'collection.replaceOne retries using operation loop' @@ -1490,6 +1641,36 @@ tests: commandName: update - commandSucceededEvent: commandName: update + - description: 'collection.replaceOne (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: replaceOne + object: *collection_retryWrites_false + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update - description: 'collection.updateOne retries using operation loop' @@ -1531,6 +1712,36 @@ tests: commandName: update - commandSucceededEvent: commandName: update + - description: 'collection.updateOne (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateOne + object: *collection_retryWrites_false + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update - description: 'collection.updateMany retries using operation loop' @@ -1572,6 +1783,36 @@ tests: commandName: update - commandSucceededEvent: commandName: update + - description: 'collection.updateMany (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateMany + object: *collection_retryWrites_false + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update - description: 'collection.findOneAndDelete retries using operation loop' @@ -1612,6 +1853,35 @@ tests: commandName: findAndModify - commandSucceededEvent: commandName: findAndModify + - description: 'collection.findOneAndDelete (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndDelete + object: *collection_retryWrites_false + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify - description: 'collection.findOneAndReplace retries using operation loop' @@ -1653,6 +1923,36 @@ tests: commandName: findAndModify - commandSucceededEvent: commandName: findAndModify + - description: 'collection.findOneAndReplace (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndReplace + object: *collection_retryWrites_false + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify - description: 'collection.findOneAndUpdate retries using operation loop' @@ -1694,6 +1994,36 @@ tests: commandName: findAndModify - commandSucceededEvent: commandName: findAndModify + - description: 'collection.findOneAndUpdate (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndUpdate + object: *collection_retryWrites_false + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify - description: 'collection.bulkWrite retries using operation loop' @@ -1736,6 +2066,37 @@ tests: commandName: insert - commandSucceededEvent: commandName: insert + - description: 'collection.bulkWrite (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: bulkWrite + object: *collection_retryWrites_false + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert - description: 'collection.createIndex retries using operation loop' @@ -1777,6 +2138,36 @@ tests: commandName: createIndexes - commandSucceededEvent: commandName: createIndexes + - description: 'collection.createIndex (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createIndex + object: *collection_retryWrites_false + arguments: + keys: { x: 11 } + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes - description: 'collection.dropIndex retries using operation loop' @@ -1822,6 +2213,40 @@ tests: commandName: dropIndexes - commandSucceededEvent: commandName: dropIndexes + - description: 'collection.dropIndex (write) does not retry if retryWrites=false' + operations: + - name: createIndex + object: *internal_collection + arguments: + keys: { x: 11 } + name: "x_11" + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndex + object: *collection_retryWrites_false + arguments: + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes - description: 'collection.dropIndexes retries using operation loop' @@ -1860,4 +2285,31 @@ tests: commandName: dropIndexes - commandSucceededEvent: commandName: dropIndexes + - description: 'collection.dropIndexes (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndexes + object: *collection_retryWrites_false + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index a5f642bed5..7f582fa74f 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -57,6 +57,24 @@ createEntities: database: *database_retryReads_false collectionName: *collection_name + - client: + id: &client_retryWrites_false client_retryWrites_false + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + ignoreCommandMonitoringEvents: [killCursors] + uriOptions: + retryWrites: false + + - database: + id: &database_retryWrites_false database_retryWrites_false + client: *client_retryWrites_false + databaseName: backpressure-db + + - collection: + id: &collection_retryWrites_false collection_retryWrites_false + database: *database_retryWrites_false + collectionName: *collection_name + initialData: - collectionName: *collection_name databaseName: *database_name @@ -154,4 +172,49 @@ tests: {% for operation in operations %} commandName: {{operation.command_name}} {%- endif %} + {%- if operation.operation_type == 'write' %} + - description: '{{operation.object}}.{{operation.operation_name}} (write) does not retry if retryWrites=false' {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + operations: {%- if operation.operation_name == "dropIndex" %} + - name: createIndex + object: *internal_collection + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: {{operation.operation_name}} + object: *{{operation.object}}_retryWrites_false + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + {%- endif %} + {% endfor -%} From 02fd863a39a83cd0fd05fe1f89faac69b8c1f3bd Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 15:29:35 -0400 Subject: [PATCH 08/11] test `runCommand` with both `retryReads=false` and `retryWrites=false` --- .../tests/backpressure-retry-loop.json | 59 +++++++++++++++++++ .../tests/backpressure-retry-loop.yml | 30 ++++++++++ .../backpressure-retry-loop.yml.template | 4 +- 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 15314ab9ff..81d8e1f641 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -1282,6 +1282,65 @@ } ] }, + { + "description": "database.runCommand (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database_retryWrites_false", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, { "description": "database.createChangeStream retries using operation loop", "operations": [ diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index dfc026a79d..f2800a0ff0 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -642,6 +642,36 @@ tests: commandName: ping - commandFailedEvent: commandName: ping + - description: 'database.runCommand (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database_retryWrites_false + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping - description: 'database.createChangeStream retries using operation loop' diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 7f582fa74f..68f982a9cc 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -136,7 +136,7 @@ tests: {% for operation in operations %} - commandSucceededEvent: commandName: {{operation.command_name}} - {%- if operation.operation_type == 'read' %} + {%- if operation.operation_type == 'read' or operation.operation_name == "runCommand" %} - description: '{{operation.object}}.{{operation.operation_name}} (read) does not retry if retryReads=false' operations: - name: failPoint @@ -172,7 +172,7 @@ tests: {% for operation in operations %} commandName: {{operation.command_name}} {%- endif %} - {%- if operation.operation_type == 'write' %} + {%- if operation.operation_type == 'write' or operation.operation_name == "runCommand" %} - description: '{{operation.object}}.{{operation.operation_name}} (write) does not retry if retryWrites=false' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 From 806d0cdd5af2a7cb3dca292b807224fa0de9251b Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 15:31:36 -0400 Subject: [PATCH 09/11] remove separate `runCommand` tests --- .../backpressure-runCommand-requirements.json | 267 ------------------ .../backpressure-runCommand-requirements.yml | 141 --------- 2 files changed, 408 deletions(-) delete mode 100644 source/client-backpressure/tests/backpressure-runCommand-requirements.json delete mode 100644 source/client-backpressure/tests/backpressure-runCommand-requirements.yml diff --git a/source/client-backpressure/tests/backpressure-runCommand-requirements.json b/source/client-backpressure/tests/backpressure-runCommand-requirements.json deleted file mode 100644 index 38d2dc56c5..0000000000 --- a/source/client-backpressure/tests/backpressure-runCommand-requirements.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "description": "tests that runCommand only retries when retryReads=true and retryWrites=true", - "schemaVersion": "1.3", - "runOnRequirements": [ - { - "minServerVersion": "4.4", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ] - } - ], - "createEntities": [ - { - "client": { - "id": "client_defaults", - "useMultipleMongoses": false, - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent", - "commandFailedEvent" - ] - } - }, - { - "database": { - "id": "database_defaults", - "client": "client_defaults", - "databaseName": "backpressure-runCommand-requirements-db" - } - }, - { - "client": { - "id": "client_retryReads_false", - "useMultipleMongoses": false, - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent", - "commandFailedEvent" - ], - "uriOptions": { - "retryReads": false - } - } - }, - { - "database": { - "id": "database_retryReads_false", - "client": "client_retryReads_false", - "databaseName": "backpressure-runCommand-requirements-db" - } - }, - { - "client": { - "id": "client_retryWrites_false", - "useMultipleMongoses": false, - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent", - "commandFailedEvent" - ], - "uriOptions": { - "retryWrites": false - } - } - }, - { - "database": { - "id": "database_retryWrites_false", - "client": "client_retryWrites_false", - "databaseName": "backpressure-runCommand-requirements-db" - } - }, - { - "client": { - "id": "internal_client", - "useMultipleMongoses": false - } - } - ], - "tests": [ - { - "description": "database.runCommand retries with defaults (retryReads=true and retryWrites=true)", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "internal_client", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 1 - }, - "data": { - "failCommands": [ - "ping" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], - "errorCode": 2 - } - } - } - }, - { - "name": "runCommand", - "object": "database_defaults", - "arguments": { - "command": { - "ping": 1 - }, - "commandName": "ping" - } - } - ], - "expectEvents": [ - { - "client": "client_defaults", - "events": [ - { - "commandStartedEvent": { - "commandName": "ping" - } - }, - { - "commandFailedEvent": { - "commandName": "ping" - } - }, - { - "commandStartedEvent": { - "commandName": "ping" - } - }, - { - "commandSucceededEvent": { - "commandName": "ping" - } - } - ] - } - ] - }, - { - "description": "database.runCommand does not retry when retryReads=false", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "internal_client", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 1 - }, - "data": { - "failCommands": [ - "ping" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], - "errorCode": 2 - } - } - } - }, - { - "name": "runCommand", - "object": "database_retryReads_false", - "arguments": { - "command": { - "ping": 1 - }, - "commandName": "ping" - }, - "expectError": { - "isError": true, - "isClientError": false - } - } - ], - "expectEvents": [ - { - "client": "client_retryReads_false", - "events": [ - { - "commandStartedEvent": { - "commandName": "ping" - } - }, - { - "commandFailedEvent": { - "commandName": "ping" - } - } - ] - } - ] - }, - { - "description": "database.runCommand does not retry when retryWrites=false", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "internal_client", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 1 - }, - "data": { - "failCommands": [ - "ping" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], - "errorCode": 2 - } - } - } - }, - { - "name": "runCommand", - "object": "database_retryWrites_false", - "arguments": { - "command": { - "ping": 1 - }, - "commandName": "ping" - }, - "expectError": { - "isError": true, - "isClientError": false - } - } - ], - "expectEvents": [ - { - "client": "client_retryWrites_false", - "events": [ - { - "commandStartedEvent": { - "commandName": "ping" - } - }, - { - "commandFailedEvent": { - "commandName": "ping" - } - } - ] - } - ] - } - ] -} diff --git a/source/client-backpressure/tests/backpressure-runCommand-requirements.yml b/source/client-backpressure/tests/backpressure-runCommand-requirements.yml deleted file mode 100644 index 599bcef85c..0000000000 --- a/source/client-backpressure/tests/backpressure-runCommand-requirements.yml +++ /dev/null @@ -1,141 +0,0 @@ -description: tests that runCommand only retries when retryReads=true and retryWrites=true - -schemaVersion: '1.3' - -runOnRequirements: - - minServerVersion: '4.4' # failCommand - topologies: [replicaset, sharded, load-balanced] - -createEntities: - - client: - id: &client_defaults client_defaults - useMultipleMongoses: false - observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - - - database: - id: &database_defaults database_defaults - client: *client_defaults - databaseName: backpressure-runCommand-requirements-db - - - client: - id: &client_retryReads_false client_retryReads_false - useMultipleMongoses: false - observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - uriOptions: - retryReads: false - - - database: - id: &database_retryReads_false database_retryReads_false - client: *client_retryReads_false - databaseName: backpressure-runCommand-requirements-db - - - client: - id: &client_retryWrites_false client_retryWrites_false - useMultipleMongoses: false - observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - uriOptions: - retryWrites: false - - - database: - id: &database_retryWrites_false database_retryWrites_false - client: *client_retryWrites_false - databaseName: backpressure-runCommand-requirements-db - - - client: - id: &internal_client internal_client - useMultipleMongoses: false - -tests: - - description: 'database.runCommand retries with defaults (retryReads=true and retryWrites=true)' - operations: - - name: failPoint - object: testRunner - arguments: - client: *internal_client - failPoint: - configureFailPoint: failCommand - mode: { times: 1 } - data: - failCommands: [ping] - errorLabels: [RetryableError, SystemOverloadedError] - errorCode: 2 - - - name: runCommand - object: *database_defaults - arguments: - command: { ping: 1 } - commandName: ping - - expectEvents: - - client: *client_defaults - events: - - commandStartedEvent: - commandName: ping - - commandFailedEvent: - commandName: ping - - commandStartedEvent: - commandName: ping - - commandSucceededEvent: - commandName: ping - - - description: 'database.runCommand does not retry when retryReads=false' - operations: - - name: failPoint - object: testRunner - arguments: - client: *internal_client - failPoint: - configureFailPoint: failCommand - mode: { times: 1 } - data: - failCommands: [ping] - errorLabels: [RetryableError, SystemOverloadedError] - errorCode: 2 - - - name: runCommand - object: *database_retryReads_false - arguments: - command: { ping: 1 } - commandName: ping - expectError: - isError: true - isClientError: false - - expectEvents: - - client: *client_retryReads_false - events: - - commandStartedEvent: - commandName: ping - - commandFailedEvent: - commandName: ping - - - description: 'database.runCommand does not retry when retryWrites=false' - operations: - - name: failPoint - object: testRunner - arguments: - client: *internal_client - failPoint: - configureFailPoint: failCommand - mode: { times: 1 } - data: - failCommands: [ping] - errorLabels: [RetryableError, SystemOverloadedError] - errorCode: 2 - - - name: runCommand - object: *database_retryWrites_false - arguments: - command: { ping: 1 } - commandName: ping - expectError: - isError: true - isClientError: false - - expectEvents: - - client: *client_retryWrites_false - events: - - commandStartedEvent: - commandName: ping - - commandFailedEvent: - commandName: ping From 1373cd8ab4d3e8240a0387eb17ba52346975ae7c Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 9 Mar 2026 15:46:05 -0400 Subject: [PATCH 10/11] test aggregate with write stage --- .../tests/backpressure-retry-loop.json | 146 ++++++++++++++++++ .../tests/backpressure-retry-loop.yml | 69 +++++++++ .../backpressure-retry-max-attempts.json | 108 +++++++++++++ .../tests/backpressure-retry-max-attempts.yml | 52 +++++++ ...enerate-backpressure-retryability-tests.py | 1 + 5 files changed, 376 insertions(+) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 81d8e1f641..0e8840f523 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -4732,6 +4732,152 @@ ] } ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [ + { + "$out": "output" + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate (write) does not retry if retryWrites=false", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection_retryWrites_false", + "arguments": { + "pipeline": [ + { + "$out": "output" + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client_retryWrites_false", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] } ] } diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index f2800a0ff0..4efcc8abbf 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -2343,3 +2343,72 @@ tests: - commandFailedEvent: commandName: dropIndexes + + - description: 'collection.aggregate retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection + arguments: + pipeline: [{$out: "output"}] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + - description: 'collection.aggregate (write) does not retry if retryWrites=false' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection_retryWrites_false + arguments: + pipeline: [{$out: "output"}] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client_retryWrites_false + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 37b610b857..1e6f46f076 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -3446,6 +3446,114 @@ ] } ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [ + { + "$out": "output" + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] } ] } diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 5ad5ec0b37..1cfabda6fd 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -1710,3 +1710,55 @@ tests: commandName: dropIndexes - commandFailedEvent: commandName: dropIndexes + + - description: 'collection.aggregate retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection + arguments: + pipeline: [{$out: "output"}] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate diff --git a/source/etc/generate-backpressure-retryability-tests.py b/source/etc/generate-backpressure-retryability-tests.py index 2a14b1bf98..342ff80f3e 100644 --- a/source/etc/generate-backpressure-retryability-tests.py +++ b/source/etc/generate-backpressure-retryability-tests.py @@ -76,6 +76,7 @@ ['keys: { x: 11 }', 'name: "x_11"'], 'write'), Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"'], 'write'), Operation('dropIndexes', 'dropIndexes', 'collection', [], 'write'), + Operation('aggregate', 'aggregate', 'collection', ['pipeline: [{$out: "output"}]'], 'write'), ] COLLECTION_OPERATIONS = COLLECTION_READ_OPERATIONS + COLLECTION_WRITE_OPERATIONS From fed5e1fbf68c4ff29f9b0dd52a521f31c7adfda6 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Tue, 10 Mar 2026 12:58:17 -0400 Subject: [PATCH 11/11] use `*database_name` To fix observed error in C driver tests: ``` error: database backpressure-db not found ``` The `*database_name` is used in initialData, creating the database before the test. --- source/client-backpressure/tests/backpressure-retry-loop.json | 2 +- source/client-backpressure/tests/backpressure-retry-loop.yml | 2 +- .../tests/backpressure-retry-loop.yml.template | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 0e8840f523..f3717ca184 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -112,7 +112,7 @@ "database": { "id": "database_retryWrites_false", "client": "client_retryWrites_false", - "databaseName": "backpressure-db" + "databaseName": "retryable-writes-tests" } }, { diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 4efcc8abbf..d29e17d707 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -68,7 +68,7 @@ createEntities: - database: id: &database_retryWrites_false database_retryWrites_false client: *client_retryWrites_false - databaseName: backpressure-db + databaseName: *database_name - collection: id: &collection_retryWrites_false collection_retryWrites_false diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 68f982a9cc..6d807b1a5d 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -68,7 +68,7 @@ createEntities: - database: id: &database_retryWrites_false database_retryWrites_false client: *client_retryWrites_false - databaseName: backpressure-db + databaseName: *database_name - collection: id: &collection_retryWrites_false collection_retryWrites_false