Use RelaxedContainerJSONEncoder & FlaskRelaxedContainerJSONProvider for json encoding where appropriate

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# This file was automatically copied from notifications-utils@121.0.0
+# This file was automatically copied from notifications-utils@124.2.0
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks

app/__init__.py

@@ -9,6 +9,7 @@
 
 from celery import current_task
 from flask import (
+    Flask,
     current_app,
     g,
     has_request_context,
@@ -27,6 +28,7 @@
 from notifications_utils.clients.signing.signing_client import Signing
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 from notifications_utils.eventlet import EventletTimeout
+from notifications_utils.json import FlaskRelaxedContainerJSONProvider
 from notifications_utils.local_vars import LazyLocalGetter
 from notifications_utils.logging import flask as utils_logging
 from sqlalchemy import event
@@ -159,9 +161,11 @@
 zendesk_client = LocalProxy(get_zendesk_client)
 
 
-def create_app(application):
+def create_app(application: Flask) -> Flask:
     from app.config import Config, configs
 
+    application.json_provider_class = FlaskRelaxedContainerJSONProvider
+
     notify_environment = os.environ["NOTIFY_ENVIRONMENT"]
 
     if notify_environment in configs:

app/celery/process_ses_receipts_tasks.py

@@ -4,6 +4,7 @@
 from celery import Task
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
@@ -98,7 +99,7 @@ def process_ses_results(  # noqa: C901
                 notification.id,
                 extra={
                     "notification_id": notification.id,
-                    "bounce_message": json.dumps(bounce_message),
+                    "bounce_message": RCJSONEncoder().encode(bounce_message),
                     "bounced_at": delivery_dt,
                     "bounced_ago": (uniform_now - delivery_dt).total_seconds() if delivery_dt is not None else None,
                     "bounce_message_type": bounce_message_bounce.get("bounceType"),

app/celery/research_mode_tasks.py

@@ -1,10 +1,10 @@
-import json
 import uuid
 from contextvars import ContextVar
 from datetime import datetime
 
 import requests
 from flask import current_app, jsonify
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.local_vars import LazyLocalGetter
 from notifications_utils.timezones import local_timezone
 from werkzeug.local import LocalProxy
@@ -80,7 +80,9 @@ def send_letter_response(notification_id: uuid.UUID, billable_units: int, postag
     data = _create_fake_letter_callback_data(notification_id, billable_units, postage)
 
     try:
-        response = requests_session.request("POST", api_call, headers=headers, data=json.dumps(data), timeout=30)  # type: ignore[attr-defined]
+        response = requests_session.request(  # type: ignore[attr-defined]
+            "POST", api_call, headers=headers, data=RCJSONEncoder().encode(data), timeout=30
+        )
         response.raise_for_status()
     except requests.HTTPError as e:
         current_app.logger.error(
@@ -179,7 +181,7 @@ def mmg_callback(notification_id, to):
     else:
         status = "3"
 
-    return json.dumps(
+    return RCJSONEncoder().encode(
         {
             "reference": "mmg_reference",
             "CID": str(notification_id),
@@ -268,7 +270,7 @@ def ses_notification_callback(reference):
         "MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
         "TopicArn": "arn:aws:sns:eu-west-1:12341234:ses_notifications",
         "Subject": None,
-        "Message": json.dumps(ses_message_body),
+        "Message": RCJSONEncoder().encode(ses_message_body),
         "Timestamp": uniform_timestamp,
         "SignatureVersion": "1",
         "Signature": "[REDACTED]",
@@ -337,7 +339,7 @@ def _ses_bounce_callback(reference, bounce_type):
         "MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
         "TopicArn": "arn:aws:sns:eu-west-1:12341234:ses_notifications",
         "Subject": None,
-        "Message": json.dumps(ses_message_body),
+        "Message": RCJSONEncoder().encode(ses_message_body),
         "Timestamp": uniform_timestamp,
         "SignatureVersion": "1",
         "Signature": "[REDACTED]",

app/celery/service_callback_tasks.py

@@ -1,10 +1,10 @@
-import json
 import logging
 from contextvars import ContextVar
 from datetime import datetime
 
 import requests
 from flask import current_app
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.local_vars import LazyLocalGetter
 from werkzeug.local import LocalProxy
 
@@ -160,7 +160,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
         response = requests_session.request(
             method="POST",
             url=service_callback_url,
-            data=json.dumps(data),
+            data=RCJSONEncoder().encode(data),
             headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
             timeout=5,
         )

app/clients/document_download.py

@@ -1,6 +1,7 @@
 import requests
 from flask import current_app, request
 from flask.ctx import has_request_context
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 
 
 class DocumentDownloadError(Exception):
@@ -61,14 +62,17 @@ def upload_document(
             if filename:
                 data["filename"] = filename
 
-            headers = {"Authorization": f"Bearer {self.auth_token}"}
+            headers = {
+                "Authorization": f"Bearer {self.auth_token}",
+                "Content-Type": "application/json",
+            }
             if has_request_context() and hasattr(request, "get_onwards_request_headers"):
                 headers.update(request.get_onwards_request_headers())
 
             response = self.requests_session.post(
                 self._get_upload_url(service_id),
                 headers=headers,
-                json=data,
+                data=RCJSONEncoder().encode(data),
             )
 
             response.raise_for_status()

app/clients/letter/dvla.py

@@ -10,6 +10,7 @@
 import jwt
 import requests
 from flask import current_app
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.recipient_validation.postal_address import PostalAddress
 from requests.adapters import HTTPAdapter
 from urllib3.util.ssl_ import create_urllib3_context
@@ -153,10 +154,13 @@ def _handle_401(e: requests.HTTPError):
         with _handle_common_dvla_errors(custom_httperror_exc_handler=_handle_401):
             response = self.session.post(
                 f"{self.base_url}/thirdparty-access/v1/authenticate",
-                json={
-                    "userName": self.dvla_username.get(),
-                    "password": self.dvla_password.get(),
-                },
+                headers={"Content-Type": "application/json"},
+                data=RCJSONEncoder().encode(
+                    {
+                        "userName": self.dvla_username.get(),
+                        "password": self.dvla_password.get(),
+                    }
+                ),
             )
             response.raise_for_status()
 
@@ -212,11 +216,14 @@ def _handle_401(e: requests.HTTPError):
             with _handle_common_dvla_errors(custom_httperror_exc_handler=_handle_401):
                 response = self.session.post(
                     f"{self.base_url}/thirdparty-access/v1/password",
-                    json={
-                        "userName": self.dvla_username.get(),
-                        "password": self.dvla_password.get(),
-                        "newPassword": new_password,
-                    },
+                    headers={"Content-Type": "application/json"},
+                    data=RCJSONEncoder().encode(
+                        {
+                            "userName": self.dvla_username.get(),
+                            "password": self.dvla_password.get(),
+                            "newPassword": new_password,
+                        }
+                    ),
                 )
                 response.raise_for_status()
 
@@ -283,16 +290,21 @@ def _handle_http_errors(e: requests.HTTPError):
         with _handle_common_dvla_errors(custom_httperror_exc_handler=_handle_http_errors):
             response = self.session.post(
                 f"{self.base_url}/print-request/v1/print/jobs",
-                headers=self._get_auth_headers(),
-                json=self._format_create_print_job_json(
-                    notification_id=notification_id,
-                    reference=reference,
-                    address=address,
-                    postage=postage,
-                    service_id=service_id,
-                    organisation_id=organisation_id,
-                    pdf_file=pdf_file,
-                    callback_url=callback_url,
+                headers={
+                    "Content-Type": "application/json",
+                    **self._get_auth_headers(),
+                },
+                data=RCJSONEncoder().encode(
+                    self._format_create_print_job_json(
+                        notification_id=notification_id,
+                        reference=reference,
+                        address=address,
+                        postage=postage,
+                        service_id=service_id,
+                        organisation_id=organisation_id,
+                        pdf_file=pdf_file,
+                        callback_url=callback_url,
+                    )
                 ),
             )
             response.raise_for_status()

app/clients/sms/mmg.py

@@ -1,6 +1,7 @@
 import json
 
 import requests
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 
 from app.clients.sms import SmsClient, SmsClientResponseException
 from app.otel_metrics.provider import record_request_duration
@@ -100,7 +101,7 @@ def try_send_sms(self, to, content, reference, international, sender):
             response = self.requests_session.request(
                 "POST",
                 self.mmg_url,
-                data=json.dumps(data),
+                data=RCJSONEncoder().encode(data),
                 headers={"Content-Type": "application/json", "Authorization": f"Basic {self.api_key}"},
                 timeout=10,
             )

app/letters/utils.py

@@ -1,12 +1,12 @@
 import io
-import json
 import math
 from datetime import datetime, timedelta
 from enum import Enum
 
 import boto3
 from flask import current_app
 from notifications_utils.clients.redis import daily_limit_cache_key
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.pdf import pdf_page_count
 from notifications_utils.s3 import s3upload
@@ -156,7 +156,7 @@ def move_scan_to_invalid_pdf_bucket(source_filename, message=None, invalid_pages
     if message:
         metadata["message"] = message
     if invalid_pages:
-        metadata["invalid_pages"] = json.dumps(invalid_pages)
+        metadata["invalid_pages"] = RCJSONEncoder().encode(invalid_pages)
     if page_count:
         metadata["page_count"] = str(page_count)
 

app/models.py

@@ -5,7 +5,7 @@
 from flask import current_app, url_for
 from jsonschema import ValidationError, validate
 from notifications_utils.insensitive_dict import InsensitiveDict
-from notifications_utils.letter_timings import get_letter_timings
+from notifications_utils.letter_timings import LetterTimings
 from notifications_utils.recipient_validation.email_address import validate_email_address
 from notifications_utils.recipient_validation.errors import InvalidRecipientError
 from notifications_utils.recipient_validation.phone_number import PhoneNumber
@@ -1824,7 +1824,7 @@ def serialize(self) -> SerializedNotification:
                 serialized["postcode"],
             ) = (personalisation.get(line) for line in address_lines_1_to_6_and_postcode_keys)
 
-            serialized["estimated_delivery"] = get_letter_timings(
+            serialized["estimated_delivery"] = LetterTimings(
                 serialized["created_at"], postage=self.postage
             ).latest_delivery.strftime(DATETIME_FORMAT)
 

app/schema_validation/__init__.py

@@ -1,9 +1,9 @@
-import json
 import re
 from datetime import datetime, timedelta
 from uuid import UUID
 
 from jsonschema import Draft7Validator, FormatChecker, ValidationError
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.recipient_validation.email_address import validate_email_address
 from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError
 from notifications_utils.recipient_validation.phone_number import PhoneNumber
@@ -144,7 +144,7 @@ def build_error_message(errors):
         fields.append({"error": "ValidationError", "message": field})
     message = {"status_code": 400, "errors": unique_errors(fields)}
 
-    return json.dumps(message)
+    return RCJSONEncoder().encode(message)
 
 
 def unique_errors(dups):

app/service/rest.py

@@ -1,10 +1,10 @@
 import itertools
-import json
 import uuid
 from datetime import datetime
 from uuid import UUID
 
 from flask import Blueprint, current_app, jsonify, request
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.letter_timings import (
     letter_can_be_cancelled,
     too_late_to_cancel_letter,
@@ -1594,7 +1594,7 @@ def create_report_request_by_type(service_id):
         extra = {
             "user_id": existing_request.user_id,
             "service_id": existing_request.service_id,
-            "report_request_parameter": json.dumps(existing_request.parameter, separators=(",", ":")),
+            "report_request_parameter": RCJSONEncoder(separators=(",", ":")).encode(existing_request.parameter),
             "report_request_id": existing_request.id,
         }
         current_app.logger.info(
@@ -1613,7 +1613,7 @@ def create_report_request_by_type(service_id):
         "report_request_id": created_request.id,
         "user_id": created_request.user_id,
         "service_id": created_request.service_id,
-        "report_request_parameter": json.dumps(created_request.parameter, separators=(",", ":")),
+        "report_request_parameter": RCJSONEncoder(separators=(",", ":")).encode(created_request.parameter),
     }
     current_app.logger.info(
         "Report request %(report_request_id)s for user %(user_id)s (service %(service_id)s) "

app/template/rest.py

@@ -5,6 +5,7 @@
 from flask import Blueprint, current_app, jsonify, request
 from flask.ctx import has_request_context
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
+from notifications_utils.json import RelaxedContainerJSONEncoder as RCJSONEncoder
 from notifications_utils.pdf import extract_page_from_pdf
 from notifications_utils.template import (
     LetterPreviewTemplate,
@@ -338,7 +339,14 @@ def _get_png_preview_or_overlaid_pdf(url, data, notification_id, json=True):
         headers.update(request.get_onwards_request_headers())
 
     if json:
-        resp = requests_post(url, json=data, headers=headers)
+        resp = requests_post(
+            url,
+            data=RCJSONEncoder().encode(data),
+            headers={
+                "Content-Type": "application/json",
+                **headers,
+            },
+        )
     else:
         resp = requests_post(url, data=data, headers=headers)