Standardized logging

log_config_template.yaml

@@ -0,0 +1,61 @@
+---
+version: 1
+disable_existing_loggers: false
+formatters:
+  default:
+    "()": aind_data_upload_utils.CustomJsonFormatter
+    format: "%(asctime)s %(levelname)s %(module)s %(lineno)s %(message)s"
+    reserved_attrs:
+      - args
+      - asctime
+      - created
+      - exc_info
+      - exc_text
+      - filename
+      - funcName
+      - levelname
+      - levelno
+      - lineno
+      - message
+      - module
+      - msecs
+      - msg
+      - name
+      - pathname
+      - process
+      - processName
+      - relativeCreated
+      - stack_info
+      - taskName
+      - thread
+      - threadName
+      - color_message
+    rename_fields:
+      "asctime": "timestamp"
+      "levelname": "level"
+    static_fields:
+      "environment": "local"
+      "software_version": ext://aind_data_upload_utils.__version__
+      "software_name": "aind-data-upload-utils"
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: default
+    level: INFO
+    stream: ext://sys.stdout
+loggers:
+  '':
+    handlers:
+    - console
+    level: INFO
+    propagate: true
+  uvicorn.error:
+    handlers:
+      - console
+    level: INFO
+    propagate: false
+  uvicorn.access:
+    handlers:
+      - console
+    level: INFO
+    propagate: false

pyproject.toml

@@ -21,7 +21,9 @@ dependencies = [
     'pydantic>2.9',
     'pydantic-settings>=2.8',
     'boto3',
-    'requests'
+    'requests',
+    'python-json-logger',
+    'PyYAML'
 ]
 
 [project.optional-dependencies]

src/aind_data_upload_utils/__init__.py

@@ -1,3 +1,43 @@
 """Init package"""
 
+import logging.config
+import os
+from datetime import datetime, timezone
+from logging import LogRecord
+
+import yaml
+from pythonjsonlogger import json as log_json
+
 __version__ = "0.18.2"
+
+
+# We want to standardize the timestamp format to UTC and ISO-8601, which
+# requires a custom formatter and can't be done through configuration only.
+class CustomJsonFormatter(log_json.JsonFormatter):
+    """Custom class to format log timestamps as ISO-8601 UTC"""
+
+    def formatTime(self, record: LogRecord, datefmt=None) -> str:
+        """
+        Format timestamp as ISO-8601 UTC
+
+        Parameters
+        ----------
+        record : LogRecord
+        datefmt : str, optional
+          Default is None. Unused parameter, kept for signature compatibility.
+
+        Returns
+        -------
+        str
+
+        """
+        dt = datetime.fromtimestamp(record.created, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+if os.path.isfile(os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml")):
+    config_path = os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml")
+    with open(config_path, "rt") as f:
+        config = yaml.safe_load(f.read())
+    logging.config.dictConfig(config)
+    logging.info(f"Found logging file at: {config_path}")

src/aind_data_upload_utils/check_directories_job.py

@@ -16,10 +16,6 @@
 from pydantic import BaseModel, Field
 from pydantic_settings import BaseSettings
 
-# Set log level from env var
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class DirectoriesToCheckConfigs(BaseModel):
     """Basic model needed from BasicUploadConfigs"""
@@ -206,12 +202,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/check_metadata_job.py

@@ -4,19 +4,13 @@
 
 import argparse
 import json
-import logging
-import os
 import sys
 from pathlib import Path
 from typing import Set, Tuple, Union
 
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
-# Set log level from env var
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(BaseSettings):
     """Job settings for CheckMetadataJob"""
@@ -117,12 +111,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/copy_metadata_files.py

@@ -4,7 +4,6 @@
 
 import argparse
 import logging
-import os
 import shutil
 import sys
 from pathlib import Path
@@ -13,10 +12,6 @@
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
-# Set log level from env var
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(BaseSettings):
     """Job settings for CheckMetadataJob"""
@@ -81,12 +76,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/create_s5_commands_job.py

@@ -18,10 +18,6 @@
 from pydantic import Field, model_validator
 from pydantic_settings import BaseSettings
 
-# Set log level from env var
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(BaseSettings):
     """Job settings for CreateS5CommandsJob"""
@@ -221,12 +217,10 @@ def run_job(self) -> None:
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/create_sym_links_job.py

@@ -9,9 +9,6 @@
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(
     BaseSettings, cli_parse_args=True, cli_ignore_unknown_args=True

src/aind_data_upload_utils/delete_folders_job.py

@@ -4,7 +4,6 @@
 
 import argparse
 import logging
-import os
 import re
 import sys
 from pathlib import Path
@@ -19,9 +18,6 @@
     DeleteStagingFolderJob,
 )
 
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(BaseSettings):
     """Job settings for DeleteFoldersJob"""
@@ -88,12 +84,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_source_folders_job.py

@@ -21,9 +21,6 @@
     DeleteStagingFolderJob,
 )
 
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class DirectoriesToDeleteConfigs(BaseModel):
     """Basic model that can be easily passed via the transfer service."""
@@ -232,12 +229,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_staging_folder_job.py

@@ -17,8 +17,6 @@
 from pydantic_settings import BaseSettings
 
 # Set log level from env var
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
-logging.basicConfig(level=LOG_LEVEL)
 
 
 class JobSettings(BaseSettings):
@@ -191,12 +189,10 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(
-            r"""
+        help=(r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """
-        ),
+            """),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/trigger_co_cleanup_notification.py

@@ -5,7 +5,6 @@
 import argparse
 import csv
 import logging
-import os
 import sys
 from collections import defaultdict
 from io import StringIO
@@ -17,9 +16,6 @@
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
-logging.basicConfig(level=LOG_LEVEL)
-
 
 class JobSettings(BaseSettings):
     """Job settings for WebhookNotificationJob"""