Ftr/dev 14453 transaction loader refactor performance

.github/workflows/code-style-checks.yaml

@@ -31,6 +31,18 @@ jobs:
       - name: Checkout Source Repository
         uses: actions/checkout@v4
 
+      - name: Get Changed Python Files
+        id: changed-python-files
+        uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62
+        with:
+          files: |
+            **.py
+          separator: " "
+
+      - name: Check If `changed-python-files` Is An Empty String
+        if: steps.changed-python-files.outputs.all_changed_files == ''
+        run: echo "No changed files detected"
+
       - name: Set Combined ENV
         run: |
           echo "DATA_BROKER_DATABASE_URL=postgres://$BROKER_DB_USER:$BROKER_DB_PASSWORD@$BROKER_DB_HOST:$BROKER_DB_PORT/$BROKER_DB_NAME" >> $GITHUB_ENV
@@ -52,11 +64,10 @@ jobs:
       - name: Init Python Environment
         uses: ./.github/actions/init-python-environment
 
-      - name: Run Flake8
-        run: flake8
-
-      - name: Run Black
-        run: black --check --diff .
+      # changed-python-files could be an empty string, which would cause `ruff check` to be run against the entire project.
+      - name: Run Ruff Linter
+        if: steps.changed-python-files.outputs.all_changed_files != ''
+        run: ruff check ${{ steps.changed-python-files.outputs.all_changed_files }}
 
       - name: Run Check For Endpoint Documentation
         run: python manage.py check_for_endpoint_documentation

.pre-commit-config.yaml

@@ -3,16 +3,9 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-    -   id: debug-statements
--   repo: https://github.com/pycqa/flake8.git
-    rev: 7.1.0
+        - id: debug-statements
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.14
     hooks:
-    -   id: flake8
-        language_version: python3.10.12
--   repo: https://github.com/psf/black
-    rev: 24.10.0
-    hooks:
-    -   id: black
-        language_version: python3.10.12
-        additional_dependencies:
-            - "click==8.0.4"
+        - id: ruff-check
+          types_or: [ python, pyi ]  # avoid linting other python file types like Jupyter notebooks

pyproject.toml

@@ -13,7 +13,7 @@ readme = 'README.md'
 license = "CC0-1.0"
 license-files = ["LICENSE"]
 requires-python = '>=3.10'
-classifiers=[
+classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
@@ -97,11 +97,9 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "black==24.10.0",
     "click==8.1.7",
     "docker==7.0.0",
     "dredd-hooks==0.2.0",
-    "flake8==7.1.0",
     "importlib-metadata==8.5.0",
     "mock==5.1.*",
     "model-bakery==1.17.*",
@@ -112,6 +110,7 @@ dev = [
     "pytest-django==4.8.*",
     "pytest-pretty==1.2.*",
     "pytest-xdist==3.5.*",
+    "ruff==0.14.14",
 ]
 spark = [
     "delta-spark==3.2.*",
@@ -138,14 +137,13 @@ DJANGO_SETTINGS_MODULE = "usaspending_api.settings"
 addopts = "--cov=usaspending_api"
 markers = [
     "signal_handling: Mark all tests that import the signal library and invoke signals. This MUST be done on the main thread, and can cause errors if pytest-xdist subordinates parellel test sessions to background threads.",
-
     # These are "auto" marked based on fixture usage. See conftest.py pytest_collection_modifyitems
     "spark: Mark all tests using the spark fixture. Can be selected with -m spark or deselected with -m (not spark)",
     "database: Mark all integration tests using a database. Can be selected with -m database or deselected with -m (not database)",
     "elasticsearch: Mark all integration tests using Elasticsearch. Can be selected with -m database or deselected with -m (not elasticsearch)",
 ]
 pythonpath = [
-  "."
+    "."
 ]
 
 [tool.coverage.run]
@@ -163,7 +161,68 @@ exclude_lines = [
     "pragma: no cover"
 ]
 
-[tool.black]
-line-length = 120
-target-version = ['py310']
-exclude = '/(\.git|\.venv|venv|migrations)/'
+[tool.ruff.lint]
+preview = true  # enable new rules
+exclude = [
+    '.git',
+    '.venv',
+    'venv',
+    '**/migrations/**',
+    'build',
+    'usaspending_api.egg-info'
+]
+
+select = [
+    "PLR0913", # max arguments in function
+    "PLR0904", # max number of public methods
+    "PLR0911", # max number of return statements
+    "PLR0916", # max number of boolean expressions
+    "PLR0915", # max number of lines in a function
+    "PLR0912", # max number of logical branches in a function
+    "PLR1702", # max number of nested blocks
+    "C901", # cognitive complexity (functions)
+    "I001", # unsorted imports
+    "B", # flake8 bugbear
+    "E", # pycodestyle errors
+    "F", # pyflakes
+    "W", # pycodestyle warnings
+    "ANN001", # missing type annotation for function argument
+    "ANN201", # missing return type annotation for public function or method
+    "ANN202", # missing return type annotation for private function or method
+]
+ignore = [
+    "E203", # whitespace before punctuation
+]
+
+pylint.max-args = 6
+pylint.max-public-methods = 20
+pylint.max-returns = 3
+pylint.max-bool-expr = 8
+pylint.max-statements = 45
+pylint.max-branches = 10
+pylint.max-nested-blocks = 5
+mccabe.max-complexity = 15
+pycodestyle.max-line-length = 120
+
+[tool.ruff.lint.per-file-ignores]
+"**/tests/**/test*.py" = [
+    "ANN001", # missing-type-function-argument; avoid conflict with fixtures
+    "ANN201", # missing-return-type-undocumented-public-function; avoid conflict with test case return values
+    "ANN202", # missing-return-type-undocumented-private-function; avoid conflict with test case return values
+    "PLR0913", # too-many-arguments; avoid conflict with too many fixtures
+    "PLR0915", # too-many-statements; avoid conflict with long fixtures
+]
+"**/conftest*.py" = [
+    "ANN001", # missing-type-function-argument; avoid conflict with fixtures
+    "ANN201", # missing-return-type-undocumented-public-function; avoid conflict with test case return values
+    "ANN202", # missing-return-type-undocumented-private-function; avoid conflict with test case return values
+    "PLR0913", # too-many-arguments; avoid conflict with too many fixtures
+    "PLR0915", # too-many-statements; avoid conflict with long fixtures
+]
+"**/tests/**/data/**.py" = [
+    "ANN001", # missing-type-function-argument; avoid conflict with fixtures
+    "ANN201", # missing-return-type-undocumented-public-function; avoid conflict with test case return values
+    "ANN202", # missing-return-type-undocumented-private-function; avoid conflict with test case return values
+    "PLR0913", # too-many-arguments; avoid conflict with too many fixtures
+    "PLR0915", # too-many-statements; avoid conflict with long fixtures
+]

usaspending_api/accounts/helpers.py

@@ -1,6 +1,5 @@
 import datetime
 
-
 TAS_COMPONENT_TO_FIELD_MAPPING = {
     "ata": "allocation_transfer_agency_id",
     "aid": "agency_id",
@@ -12,7 +11,9 @@
 }
 
 
-def start_and_end_dates_from_fyq(fiscal_year, fiscal_quarter):
+def start_and_end_dates_from_fyq(
+    fiscal_year: int, fiscal_quarter: int
+) -> tuple[datetime.date, datetime.date]:
     if fiscal_quarter == 1:
         start_date = datetime.date(fiscal_year - 1, 10, 1)
         end_date = datetime.date(fiscal_year - 1, 12, 31)

usaspending_api/awards/delta_models/awards.py

@@ -17,7 +17,7 @@
     "generated_unique_award_id": "STRING NOT NULL",
     "id": "LONG",
     "is_fpds": "BOOLEAN NOT NULL",
-    "last_modified_date": "DATE",
+    "last_modified_date": "TIMESTAMP",
     "latest_transaction_id": "LONG",
     "non_federal_funding_amount": "NUMERIC(23,2)",
     "officer_1_amount": "NUMERIC(23,2)",
@@ -55,4 +55,5 @@
     )
     USING DELTA
     LOCATION 's3a://{{SPARK_S3_BUCKET}}/{{DELTA_LAKE_S3_PATH}}/{{DESTINATION_DATABASE}}/{{DESTINATION_TABLE}}'
+    TBLPROPERTIES (delta.enableChangeDataFeed = true)
 """

usaspending_api/awards/migrations/0115_alter_transactionnormalized_last_modified_date.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("awards", "0114_alter_ctodlinkageupdates_award_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transactionnormalized",
+            name="last_modified_date",
+            field=models.DateTimeField(null=True),
+        ),
+    ]

usaspending_api/awards/models/transaction_normalized.py

@@ -1,14 +1,15 @@
 import os
 
-from django.db import models
-
 from django.contrib.postgres.fields import ArrayField
+from django.db import models
 
 
 class TransactionNormalized(models.Model):
     id = models.BigAutoField(primary_key=True)
     award = models.ForeignKey(
-        "search.AwardSearch", on_delete=models.DO_NOTHING, help_text="The award which this transaction is contained in"
+        "search.AwardSearch",
+        on_delete=models.DO_NOTHING,
+        help_text="The award which this transaction is contained in",
     )
     usaspending_unique_transaction_id = models.TextField(
         blank=True,
@@ -29,17 +30,25 @@ class TransactionNormalized(models.Model):
         help_text="The plain text description of the transaction type",
     )
     period_of_performance_start_date = models.DateField(
-        verbose_name="Period of Performance Start Date", null=True, help_text="The period of performance start date"
+        verbose_name="Period of Performance Start Date",
+        null=True,
+        help_text="The period of performance start date",
     )
     period_of_performance_current_end_date = models.DateField(
         verbose_name="Period of Performance Current End Date",
         null=True,
         help_text="The current end date of the period of performance",
     )
     action_date = models.DateField(
-        verbose_name="Transaction Date", help_text="The date this transaction was actioned", db_index=True
+        verbose_name="Transaction Date",
+        help_text="The date this transaction was actioned",
+        db_index=True,
+    )
+    action_type = models.TextField(
+        blank=True,
+        null=True,
+        help_text="The type of transaction. For example, A, B, C, D",
     )
-    action_type = models.TextField(blank=True, null=True, help_text="The type of transaction. For example, A, B, C, D")
     action_type_description = models.TextField(blank=True, null=True)
     federal_action_obligation = models.DecimalField(
         max_digits=23,
@@ -90,22 +99,36 @@ class TransactionNormalized(models.Model):
         null=True,
         help_text="The agency which is funding this transaction",
     )
-    description = models.TextField(null=True, help_text="The description of this transaction")
-    last_modified_date = models.DateField(
+    description = models.TextField(
+        null=True, help_text="The description of this transaction"
+    )
+    last_modified_date = models.DateTimeField(
         blank=True, null=True, help_text="The date this transaction was last modified"
     )
-    certified_date = models.DateField(blank=True, null=True, help_text="The date this transaction was certified")
+    certified_date = models.DateField(
+        blank=True, null=True, help_text="The date this transaction was certified"
+    )
     create_date = models.DateTimeField(
-        auto_now_add=True, blank=True, null=True, help_text="The date this transaction was created in the API"
+        auto_now_add=True,
+        blank=True,
+        null=True,
+        help_text="The date this transaction was created in the API",
     )
     update_date = models.DateTimeField(
-        auto_now=True, null=True, help_text="The last time this transaction was updated in the API", db_index=True
+        auto_now=True,
+        null=True,
+        help_text="The last time this transaction was updated in the API",
+        db_index=True,
+    )
+    fiscal_year = models.IntegerField(
+        blank=True, null=True, help_text="Fiscal Year calculated based on Action Date"
     )
-    fiscal_year = models.IntegerField(blank=True, null=True, help_text="Fiscal Year calculated based on Action Date")
     transaction_unique_id = models.TextField(
         blank=False, null=False, default="NONE", verbose_name="Transaction Unique ID"
     )
-    is_fpds = models.BooleanField(blank=False, null=False, default=False, verbose_name="Is FPDS")
+    is_fpds = models.BooleanField(
+        blank=False, null=False, default=False, verbose_name="Is FPDS"
+    )
     funding_amount = models.DecimalField(
         max_digits=23,
         decimal_places=2,
@@ -114,7 +137,11 @@ class TransactionNormalized(models.Model):
         help_text="Assistance data variable.  non_federal_funding_amount + federal_action_obligation",
     )
     non_federal_funding_amount = models.DecimalField(
-        max_digits=23, decimal_places=2, blank=True, null=True, help_text="Assistance Data variable."
+        max_digits=23,
+        decimal_places=2,
+        blank=True,
+        null=True,
+        help_text="Assistance Data variable.",
     )
     unique_award_key = models.TextField(null=True, db_index=True)  # From broker.
     business_categories = ArrayField(models.TextField(), default=list)
@@ -149,9 +176,18 @@ class Meta:
 vw_transaction_normalized_sql = f"""
     CREATE OR REPLACE VIEW rpt.vw_transaction_normalized AS
         SELECT
-            {(','+os.linesep+' '*12).join([
-                (v+(f'::{NORM_CASTED_COL_MAP[k]}' if k in NORM_CASTED_COL_MAP else '')).ljust(62)+' AS '+k.ljust(48)
-                for k, v in NORM_TO_TRANSACTION_SEARCH_COL_MAP.items()])}
+            {
+    ("," + os.linesep + " " * 12).join(
+        [
+            (
+                v + (f"::{NORM_CASTED_COL_MAP[k]}" if k in NORM_CASTED_COL_MAP else "")
+            ).ljust(62)
+            + " AS "
+            + k.ljust(48)
+            for k, v in NORM_TO_TRANSACTION_SEARCH_COL_MAP.items()
+        ]
+    )
+}
         FROM
             rpt.transaction_search;
 """