Migrate to async juriscraper

.github/workflows/lint.yml

@@ -82,4 +82,7 @@ jobs:
           cl/search/docket_number_cleaner.py \
           cl/search/management/commands/clean_docket_number_raw.py \
           cl/scrapers/management/commands/back_scrape_dockets.py \
+          cl/corpus_importer/management/commands/import_texas_dockets.py \
+          cl/corpus_importer/management/utils.py \
+          cl/lib/decorators.py \
           cl/corpus_importer/management/commands/import_scotus_dockets.py

cl/alerts/management/commands/handle_old_docket_alerts.py

@@ -1,5 +1,6 @@
 from argparse import RawTextHelpFormatter
 
+from asgiref.sync import async_to_sync, sync_to_async
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.mail import EmailMultiAlternatives
@@ -78,7 +79,7 @@ def build_user_report(user, delete=False):
     return report
 
 
-def send_old_alert_warning_email_and_webhook(user, report) -> int:
+async def send_old_alert_warning_email_and_webhook(user, report) -> int:
     """Send alerts emails and webhooks for old alerts
 
     :param user: The user with terminated dockets
@@ -91,17 +92,21 @@ def send_old_alert_warning_email_and_webhook(user, report) -> int:
     )
     webhook_count = 0
     if report.very_old_alerts or report.disabled_alerts:
-        for user_webhook in user_webhooks:
-            send_old_alerts_webhook_event(user_webhook, report)
+        async for user_webhook in user_webhooks:
+            await send_old_alerts_webhook_event(user_webhook, report)
             webhook_count += 1
 
     count = report.total_count()
     subject_template = loader.get_template("emails/old_email_subject.txt")
     subject = subject_template.render({"count": count}).strip()
-    txt = loader.get_template("emails/old_alert_email.txt").render(
+    txt = await sync_to_async(
+        loader.get_template("emails/old_alert_email.txt").render
+    )(
         {"report_data": report},
     )
-    html = loader.get_template("emails/old_alert_email.html").render(
+    html = await sync_to_async(
+        loader.get_template("emails/old_alert_email.html").render
+    )(
         {"report_data": report},
     )
     msg = EmailMultiAlternatives(
@@ -174,9 +179,9 @@ def handle(self, *args, **options):
             count = report.total_count()
             if options["send_alerts"] and count > 0:
                 emails_sent += 1
-                webhooks_count = send_old_alert_warning_email_and_webhook(
-                    user, report
-                )
+                webhooks_count = async_to_sync(
+                    send_old_alert_warning_email_and_webhook
+                )(user, report)
                 webhooks_sent += webhooks_count
 
         logger.info(

cl/alerts/management/commands/monitor_pacer.py

@@ -1,6 +1,7 @@
 import datetime
 import time
 
+from asgiref.sync import async_to_sync
 from django.conf import settings
 from django.core.mail import send_mail
 from django.template import loader
@@ -48,12 +49,12 @@ def handle(self, *args, **options):
         s = ProxyPacerSession(
             username=settings.PACER_USERNAME, password=settings.PACER_PASSWORD
         )
-        s.login()
+        async_to_sync(s.login)()
         report = CaseQueryAdvancedBankruptcy("canb", s)
         t1 = now()
         while True:
             query = "Pacific"
-            report.query(
+            async_to_sync(report.query)(
                 name_last=query,
                 filed_from=datetime.date(2019, 1, 28),
                 filed_to=datetime.date(2019, 1, 30),
@@ -66,7 +67,7 @@ def handle(self, *args, **options):
                 exit(0)
 
             query = "PG&E"
-            report.query(
+            async_to_sync(report.query)(
                 name_last=query,
                 filed_from=datetime.date(2019, 1, 28),
                 filed_to=datetime.date(2019, 1, 30),
@@ -83,5 +84,5 @@ def handle(self, *args, **options):
             min_login_frequency = 60 * 30  # thirty minutes
             if (t2 - t1).seconds > min_login_frequency:
                 print("Logging in again.")
-                s.login()
+                async_to_sync(s.login)()
                 t1 = now()

cl/api/utils.py

@@ -7,6 +7,7 @@
 from typing import Any, TypedDict
 
 import eyecite
+from asgiref.sync import async_to_sync, sync_to_async
 from dateutil import parser
 from dateutil.rrule import DAILY, rrule
 from django.conf import settings
@@ -25,7 +26,7 @@
 from django.views.decorators.vary import vary_on_headers
 from django_ratelimit.core import get_header
 from eyecite.tokenizers import HyperscanTokenizer
-from requests import Response
+from httpx import Response
 from rest_framework import serializers
 from rest_framework.exceptions import Throttled, ValidationError
 from rest_framework.metadata import SimpleMetadata
@@ -1185,7 +1186,7 @@ def get_next_webhook_retry_date(retry_counter: int) -> datetime:
 WEBHOOK_MAX_RETRY_COUNTER = 7
 
 
-def check_webhook_failure_count_and_notify(
+async def check_webhook_failure_count_and_notify(
     webhook_event: WebhookEvent,
 ) -> None:
     """Check if a Webhook needs to be disabled and/or send a notification about
@@ -1208,7 +1209,7 @@ def check_webhook_failure_count_and_notify(
         6: False,
         7: True,  # Send webhook disabled notification
     }
-    webhook = webhook_event.webhook
+    webhook = await Webhook.objects.aget(pk=webhook_event.webhook_id)
     if not webhook.enabled or webhook_event.debug:
         return
 
@@ -1218,33 +1219,33 @@ def check_webhook_failure_count_and_notify(
     current_try_counter = webhook_event.retry_counter
     notify = notify_on[current_try_counter]
     if notify:
-        oldest_enqueued_for_retry = WebhookEvent.objects.filter(
-            webhook=webhook_event.webhook,
+        oldest_enqueued_for_retry = await WebhookEvent.objects.filter(
+            webhook=webhook,
             event_status=WEBHOOK_EVENT_STATUS.ENQUEUED_RETRY,
             debug=False,
-        ).earliest("date_created")
+        ).aearliest("date_created")
         if current_try_counter >= WEBHOOK_MAX_RETRY_COUNTER:
             webhook.enabled = False
             update_fields.append("enabled")
             update_fields.append("date_modified")
             # If the parent webhook is disabled mark all current ENQUEUED_RETRY
             # events as ENDPOINT_DISABLED
-            WebhookEvent.objects.filter(
-                webhook=webhook_event.webhook,
+            await WebhookEvent.objects.filter(
+                webhook=webhook,
                 event_status=WEBHOOK_EVENT_STATUS.ENQUEUED_RETRY,
                 debug=False,
-            ).update(
+            ).aupdate(
                 event_status=WEBHOOK_EVENT_STATUS.ENDPOINT_DISABLED,
                 date_modified=now(),
             )
         if oldest_enqueued_for_retry.pk == webhook_event.pk:
             failure_counter = current_try_counter + 1
-            notify_failing_webhook.delay(
+            await sync_to_async(notify_failing_webhook.delay)(
                 webhook_event.pk, failure_counter, webhook.enabled
             )
 
     # Save webhook and avoid emailing admins via signal in cl.users.signals
-    webhook.save(update_fields=update_fields)
+    await webhook.asave(update_fields=update_fields)
 
 
 def update_webhook_event_after_request(
@@ -1288,7 +1289,7 @@ def update_webhook_event_after_request(
         if error is None:
             error = ""
         webhook_event.error_message = error
-        check_webhook_failure_count_and_notify(webhook_event)
+        async_to_sync(check_webhook_failure_count_and_notify)(webhook_event)
         if webhook_event.retry_counter >= WEBHOOK_MAX_RETRY_COUNTER:
             # If the webhook has reached the max retry counter, mark as failed
             webhook_event.event_status = WEBHOOK_EVENT_STATUS.FAILED

cl/api/webhooks.py

@@ -2,7 +2,9 @@
 import random
 
 import requests
+from asgiref.sync import sync_to_async
 from django.conf import settings
+from django.contrib.auth.models import User
 from elasticsearch_dsl.response import Response
 from rest_framework.renderers import JSONRenderer
 
@@ -80,7 +82,7 @@ def send_webhook_event(
         update_webhook_event_after_request(webhook_event, error=error_str)
 
 
-def send_old_alerts_webhook_event(
+async def send_old_alerts_webhook_event(
     webhook: Webhook, report: OldAlertReport
 ) -> None:
     """Send webhook event for old alerts
@@ -115,14 +117,14 @@ def send_old_alerts_webhook_event(
         post_content,
         accepted_media_type="application/json;",
     )
-    webhook_event = WebhookEvent.objects.create(
+    webhook_event = await WebhookEvent.objects.acreate(
         webhook=webhook,
         content=post_content,
     )
-    send_webhook_event(webhook_event, json_bytes)
+    await sync_to_async(send_webhook_event)(webhook_event, json_bytes)
 
 
-def send_recap_fetch_webhooks(fq: PacerFetchQueue) -> None:
+async def send_recap_fetch_webhooks(fq: PacerFetchQueue) -> None:
     """Send webhook event for processed PacerFetchQueue objects.
 
     :param fq: The PacerFetchQueue object related to the event.
@@ -137,10 +139,11 @@ def send_recap_fetch_webhooks(fq: PacerFetchQueue) -> None:
         PROCESSING_STATUS.INVALID_CONTENT,
         PROCESSING_STATUS.NEEDS_INFO,
     ]:
-        user_webhooks = fq.user.webhooks.filter(
+        user = await User.objects.aget(pk=fq.user_id)
+        user_webhooks = user.webhooks.filter(
             event_type=WebhookEventType.RECAP_FETCH, enabled=True
         )
-        for webhook in user_webhooks:
+        async for webhook in user_webhooks:
             payload = PacerFetchQueueSerializer(fq).data
             post_content = {
                 "webhook": generate_webhook_key_content(webhook),
@@ -151,11 +154,11 @@ def send_recap_fetch_webhooks(fq: PacerFetchQueue) -> None:
                 post_content,
                 accepted_media_type="application/json;",
             )
-            webhook_event = WebhookEvent.objects.create(
+            webhook_event = await WebhookEvent.objects.acreate(
                 webhook=webhook,
                 content=post_content,
             )
-            send_webhook_event(webhook_event, json_bytes)
+            await sync_to_async(send_webhook_event)(webhook_event, json_bytes)
 
 
 def send_search_alert_webhook(

cl/corpus_importer/bulk_utils.py

@@ -1,3 +1,4 @@
+from asgiref.sync import async_to_sync
 from celery import chain
 
 from cl.corpus_importer.tasks import get_pacer_doc_by_rd
@@ -45,7 +46,7 @@ def get_petitions(
     session = ProxyPacerSession(
         username=pacer_username, password=pacer_password
     )
-    session.login()
+    async_to_sync(session.login)()
     for i, rd_pk in enumerate(rds):
         if i < options["offset"]:
             i += 1
@@ -57,7 +58,7 @@ def get_petitions(
             session = ProxyPacerSession(
                 username=pacer_username, password=pacer_password
             )
-            session.login()
+            async_to_sync(session.login)()
             logger.info(f"Sent {i} tasks to celery so far.")
         logger.info("Doing row %s", i)
         throttle.maybe_wait()

cl/corpus_importer/management/commands/760_project.py

@@ -2,6 +2,7 @@
 import csv
 import os
 
+from asgiref.sync import async_to_sync
 from celery.canvas import chain
 from django.conf import settings
 
@@ -34,7 +35,7 @@ def get_dockets(options):
     session = ProxyPacerSession(
         username=PACER_USERNAME, password=PACER_PASSWORD
     )
-    session.login()
+    async_to_sync(session.login)()
     session_data = SessionData(session.cookies, session.proxy_address)
     for i, row in enumerate(reader):
         if i < options["offset"]:
@@ -101,7 +102,7 @@ def get_att_pages(options):
     session = ProxyPacerSession(
         username=PACER_USERNAME, password=PACER_PASSWORD
     )
-    session.login()
+    async_to_sync(session.login)()
     get_district_attachment_pages(
         options=options, rd_pks=rd_pks, tag_names=[TAG], session=session
     )

cl/corpus_importer/management/commands/adelman_david.py

@@ -1,6 +1,7 @@
 import csv
 import os
 
+from asgiref.sync import async_to_sync
 from celery.canvas import chain
 from django.conf import settings
 
@@ -31,7 +32,7 @@ def download_dockets(options):
     session = ProxyPacerSession(
         username=PACER_USERNAME, password=PACER_PASSWORD
     )
-    session.login()
+    async_to_sync(session.login)()
     session_data = SessionData(session.cookies, session.proxy_address)
     for i, row in enumerate(reader):
         if i < options["offset"]:

cl/corpus_importer/management/commands/buchwald_project.py

@@ -1,6 +1,7 @@
 import os
 from argparse import RawTextHelpFormatter
 
+from asgiref.sync import async_to_sync
 from celery import chain
 from django.conf import settings
 
@@ -36,7 +37,7 @@ def add_all_nysd_to_cl(options):
     session = ProxyPacerSession(
         username=PACER_USERNAME, password=PACER_PASSWORD
     )
-    session.login()
+    async_to_sync(session.login)()
 
     # IDs obtained by binary search of docket numbers on PACER website.
     earliest_id = 405990
@@ -53,7 +54,7 @@ def add_all_nysd_to_cl(options):
             session = ProxyPacerSession(
                 username=PACER_USERNAME, password=PACER_PASSWORD
             )
-            session.login()
+            async_to_sync(session.login)()
 
         throttle.maybe_wait()
         logger.info("Doing pacer_case_id: %s", pacer_case_id)
@@ -72,7 +73,7 @@ def get_dockets(options):
     session = ProxyPacerSession(
         username=PACER_USERNAME, password=PACER_PASSWORD
     )
-    session.login()
+    async_to_sync(session.login)()
 
     buchwald_id = 450
     ds = (
@@ -95,7 +96,7 @@ def get_dockets(options):
             session = ProxyPacerSession(
                 username=PACER_USERNAME, password=PACER_PASSWORD
             )
-            session.login()
+            async_to_sync(session.login)()
 
         throttle.maybe_wait()
         logger.info("%s: Doing docket with pk: %s", i, d.pk)

cl/corpus_importer/management/commands/claims_activity_project.py

@@ -6,6 +6,7 @@
 from datetime import date
 
 import pandas as pd
+from asgiref.sync import async_to_sync
 from django.conf import settings
 from juriscraper.pacer import ClaimsActivity
 
@@ -45,7 +46,7 @@ def query_and_parse_claims_activity(
     }
 
     s = ProxyPacerSession(username=PACER_USERNAME, password=PACER_PASSWORD)
-    s.login()
+    async_to_sync(s.login)()
     for court_id in courts:
         court = map_cl_to_pacer_id(court_id)
         for alias, creditor_name in creditor_names.items():