From e618997050720ab4db244d2048cb0c6dad42aed6 Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Fri, 1 May 2026 14:11:33 -0700 Subject: [PATCH 1/3] use centralized logging config --- log_config_template.yaml | 0 pyproject.toml | 4 +- src/aind_data_upload_utils/__init__.py | 38 +++++++++++++++++++ .../check_directories_job.py | 4 +- .../check_metadata_job.py | 4 +- .../copy_metadata_files.py | 3 -- .../create_s5_commands_job.py | 4 +- .../create_sym_links_job.py | 3 +- .../delete_folders_job.py | 3 +- .../delete_source_folders_job.py | 3 +- .../delete_staging_folder_job.py | 3 +- .../trigger_co_cleanup_notification.py | 3 +- 12 files changed, 49 insertions(+), 23 deletions(-) create mode 100644 log_config_template.yaml diff --git a/log_config_template.yaml b/log_config_template.yaml new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index a66e54e..cec1eba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,9 @@ dependencies = [ 'pydantic>2.9', 'pydantic-settings>=2.8', 'boto3', - 'requests' + 'requests', + 'python-json-logger', + 'PyYAML' ] [project.optional-dependencies] diff --git a/src/aind_data_upload_utils/__init__.py b/src/aind_data_upload_utils/__init__.py index 63ce3dd..d5b180d 100644 --- a/src/aind_data_upload_utils/__init__.py +++ b/src/aind_data_upload_utils/__init__.py @@ -1,3 +1,41 @@ """Init package""" +import logging.config +import os +from datetime import datetime, timezone +from logging import LogRecord + +import yaml +from pythonjsonlogger import json as log_json __version__ = "0.18.2" + +# We want to standardize the timestamp format to UTC and ISO-8601, which +# requires a custom formatter and can't be done through configuration only. +class CustomJsonFormatter(log_json.JsonFormatter): + """Custom class to format log timestamps as ISO-8601 UTC""" + + def formatTime(self, record: LogRecord, datefmt=None) -> str: + """ + Format timestamp as ISO-8601 UTC + + Parameters + ---------- + record : LogRecord + datefmt : str, optional + Default is None. Unused parameter, kept for signature compatibility. + + Returns + ------- + str + + """ + dt = datetime.fromtimestamp(record.created, tz=timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +if os.path.isfile(os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml")): + config_path = os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml") + with open(config_path, "rt") as f: + config = yaml.safe_load(f.read()) + logging.config.dictConfig(config) + logging.info(f"Found logging file at: {config_path}") \ No newline at end of file diff --git a/src/aind_data_upload_utils/check_directories_job.py b/src/aind_data_upload_utils/check_directories_job.py index d2f086d..6b393ef 100644 --- a/src/aind_data_upload_utils/check_directories_job.py +++ b/src/aind_data_upload_utils/check_directories_job.py @@ -16,9 +16,7 @@ from pydantic import BaseModel, Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) + class DirectoriesToCheckConfigs(BaseModel): diff --git a/src/aind_data_upload_utils/check_metadata_job.py b/src/aind_data_upload_utils/check_metadata_job.py index 05a466e..ec170d5 100644 --- a/src/aind_data_upload_utils/check_metadata_job.py +++ b/src/aind_data_upload_utils/check_metadata_job.py @@ -13,9 +13,7 @@ from pydantic import Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings(BaseSettings): diff --git a/src/aind_data_upload_utils/copy_metadata_files.py b/src/aind_data_upload_utils/copy_metadata_files.py index 56795cd..0533003 100644 --- a/src/aind_data_upload_utils/copy_metadata_files.py +++ b/src/aind_data_upload_utils/copy_metadata_files.py @@ -13,9 +13,6 @@ from pydantic import Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) class JobSettings(BaseSettings): diff --git a/src/aind_data_upload_utils/create_s5_commands_job.py b/src/aind_data_upload_utils/create_s5_commands_job.py index 3ba85a1..dbf3b56 100644 --- a/src/aind_data_upload_utils/create_s5_commands_job.py +++ b/src/aind_data_upload_utils/create_s5_commands_job.py @@ -18,9 +18,7 @@ from pydantic import Field, model_validator from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings(BaseSettings): diff --git a/src/aind_data_upload_utils/create_sym_links_job.py b/src/aind_data_upload_utils/create_sym_links_job.py index bdd1cb7..5cf7e10 100644 --- a/src/aind_data_upload_utils/create_sym_links_job.py +++ b/src/aind_data_upload_utils/create_sym_links_job.py @@ -9,8 +9,7 @@ from pydantic import Field from pydantic_settings import BaseSettings -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings( diff --git a/src/aind_data_upload_utils/delete_folders_job.py b/src/aind_data_upload_utils/delete_folders_job.py index 84e463e..3b3cf64 100644 --- a/src/aind_data_upload_utils/delete_folders_job.py +++ b/src/aind_data_upload_utils/delete_folders_job.py @@ -19,8 +19,7 @@ DeleteStagingFolderJob, ) -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings(BaseSettings): diff --git a/src/aind_data_upload_utils/delete_source_folders_job.py b/src/aind_data_upload_utils/delete_source_folders_job.py index aacbbaf..117b170 100644 --- a/src/aind_data_upload_utils/delete_source_folders_job.py +++ b/src/aind_data_upload_utils/delete_source_folders_job.py @@ -21,8 +21,7 @@ DeleteStagingFolderJob, ) -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) + class DirectoriesToDeleteConfigs(BaseModel): diff --git a/src/aind_data_upload_utils/delete_staging_folder_job.py b/src/aind_data_upload_utils/delete_staging_folder_job.py index b2e7634..72095ca 100644 --- a/src/aind_data_upload_utils/delete_staging_folder_job.py +++ b/src/aind_data_upload_utils/delete_staging_folder_job.py @@ -17,8 +17,7 @@ from pydantic_settings import BaseSettings # Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings(BaseSettings): diff --git a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py index f983bd3..df7875d 100644 --- a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py +++ b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py @@ -17,8 +17,7 @@ from pydantic import Field from pydantic_settings import BaseSettings -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) + class JobSettings(BaseSettings): From aaed5dc97dea97ad390cb59daf6e1cf3c0e1e332 Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Fri, 1 May 2026 14:45:38 -0700 Subject: [PATCH 2/3] linters --- src/aind_data_upload_utils/__init__.py | 4 +++- src/aind_data_upload_utils/check_directories_job.py | 8 ++------ src/aind_data_upload_utils/check_metadata_job.py | 10 ++-------- src/aind_data_upload_utils/copy_metadata_files.py | 8 ++------ src/aind_data_upload_utils/create_s5_commands_job.py | 8 ++------ src/aind_data_upload_utils/create_sym_links_job.py | 2 -- src/aind_data_upload_utils/delete_folders_job.py | 9 ++------- .../delete_source_folders_job.py | 8 ++------ .../delete_staging_folder_job.py | 7 ++----- .../trigger_co_cleanup_notification.py | 3 --- 10 files changed, 17 insertions(+), 50 deletions(-) diff --git a/src/aind_data_upload_utils/__init__.py b/src/aind_data_upload_utils/__init__.py index d5b180d..e38b640 100644 --- a/src/aind_data_upload_utils/__init__.py +++ b/src/aind_data_upload_utils/__init__.py @@ -7,8 +7,10 @@ import yaml from pythonjsonlogger import json as log_json + __version__ = "0.18.2" + # We want to standardize the timestamp format to UTC and ISO-8601, which # requires a custom formatter and can't be done through configuration only. class CustomJsonFormatter(log_json.JsonFormatter): @@ -38,4 +40,4 @@ def formatTime(self, record: LogRecord, datefmt=None) -> str: with open(config_path, "rt") as f: config = yaml.safe_load(f.read()) logging.config.dictConfig(config) - logging.info(f"Found logging file at: {config_path}") \ No newline at end of file + logging.info(f"Found logging file at: {config_path}") diff --git a/src/aind_data_upload_utils/check_directories_job.py b/src/aind_data_upload_utils/check_directories_job.py index 6b393ef..8688b01 100644 --- a/src/aind_data_upload_utils/check_directories_job.py +++ b/src/aind_data_upload_utils/check_directories_job.py @@ -17,8 +17,6 @@ from pydantic_settings import BaseSettings - - class DirectoriesToCheckConfigs(BaseModel): """Basic model needed from BasicUploadConfigs""" @@ -204,12 +202,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/check_metadata_job.py b/src/aind_data_upload_utils/check_metadata_job.py index ec170d5..4cfc986 100644 --- a/src/aind_data_upload_utils/check_metadata_job.py +++ b/src/aind_data_upload_utils/check_metadata_job.py @@ -4,8 +4,6 @@ import argparse import json -import logging -import os import sys from pathlib import Path from typing import Set, Tuple, Union @@ -14,8 +12,6 @@ from pydantic_settings import BaseSettings - - class JobSettings(BaseSettings): """Job settings for CheckMetadataJob""" @@ -115,12 +111,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/copy_metadata_files.py b/src/aind_data_upload_utils/copy_metadata_files.py index 0533003..330cd66 100644 --- a/src/aind_data_upload_utils/copy_metadata_files.py +++ b/src/aind_data_upload_utils/copy_metadata_files.py @@ -4,7 +4,6 @@ import argparse import logging -import os import shutil import sys from pathlib import Path @@ -14,7 +13,6 @@ from pydantic_settings import BaseSettings - class JobSettings(BaseSettings): """Job settings for CheckMetadataJob""" @@ -78,12 +76,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/create_s5_commands_job.py b/src/aind_data_upload_utils/create_s5_commands_job.py index dbf3b56..aedd550 100644 --- a/src/aind_data_upload_utils/create_s5_commands_job.py +++ b/src/aind_data_upload_utils/create_s5_commands_job.py @@ -19,8 +19,6 @@ from pydantic_settings import BaseSettings - - class JobSettings(BaseSettings): """Job settings for CreateS5CommandsJob""" @@ -219,12 +217,10 @@ def run_job(self) -> None: "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/create_sym_links_job.py b/src/aind_data_upload_utils/create_sym_links_job.py index 5cf7e10..64443a8 100644 --- a/src/aind_data_upload_utils/create_sym_links_job.py +++ b/src/aind_data_upload_utils/create_sym_links_job.py @@ -10,8 +10,6 @@ from pydantic_settings import BaseSettings - - class JobSettings( BaseSettings, cli_parse_args=True, cli_ignore_unknown_args=True ): diff --git a/src/aind_data_upload_utils/delete_folders_job.py b/src/aind_data_upload_utils/delete_folders_job.py index 3b3cf64..d2a992a 100644 --- a/src/aind_data_upload_utils/delete_folders_job.py +++ b/src/aind_data_upload_utils/delete_folders_job.py @@ -4,7 +4,6 @@ import argparse import logging -import os import re import sys from pathlib import Path @@ -20,8 +19,6 @@ ) - - class JobSettings(BaseSettings): """Job settings for DeleteFoldersJob""" @@ -87,12 +84,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_source_folders_job.py b/src/aind_data_upload_utils/delete_source_folders_job.py index 117b170..55f141f 100644 --- a/src/aind_data_upload_utils/delete_source_folders_job.py +++ b/src/aind_data_upload_utils/delete_source_folders_job.py @@ -22,8 +22,6 @@ ) - - class DirectoriesToDeleteConfigs(BaseModel): """Basic model that can be easily passed via the transfer service.""" @@ -231,12 +229,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_staging_folder_job.py b/src/aind_data_upload_utils/delete_staging_folder_job.py index 72095ca..3c0910e 100644 --- a/src/aind_data_upload_utils/delete_staging_folder_job.py +++ b/src/aind_data_upload_utils/delete_staging_folder_job.py @@ -19,7 +19,6 @@ # Set log level from env var - class JobSettings(BaseSettings): """Job settings for DeleteStagingFolderJob""" @@ -190,12 +189,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py index df7875d..9b17e00 100644 --- a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py +++ b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py @@ -5,7 +5,6 @@ import argparse import csv import logging -import os import sys from collections import defaultdict from io import StringIO @@ -18,8 +17,6 @@ from pydantic_settings import BaseSettings - - class JobSettings(BaseSettings): """Job settings for WebhookNotificationJob""" From abac740213a5ca82fcddc5e2d9dd23a5d070faed Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Fri, 1 May 2026 15:15:10 -0700 Subject: [PATCH 3/3] commit unsaved change to template --- log_config_template.yaml | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/log_config_template.yaml b/log_config_template.yaml index e69de29..b24e24a 100644 --- a/log_config_template.yaml +++ b/log_config_template.yaml @@ -0,0 +1,61 @@ +--- +version: 1 +disable_existing_loggers: false +formatters: + default: + "()": aind_data_upload_utils.CustomJsonFormatter + format: "%(asctime)s %(levelname)s %(module)s %(lineno)s %(message)s" + reserved_attrs: + - args + - asctime + - created + - exc_info + - exc_text + - filename + - funcName + - levelname + - levelno + - lineno + - message + - module + - msecs + - msg + - name + - pathname + - process + - processName + - relativeCreated + - stack_info + - taskName + - thread + - threadName + - color_message + rename_fields: + "asctime": "timestamp" + "levelname": "level" + static_fields: + "environment": "local" + "software_version": ext://aind_data_upload_utils.__version__ + "software_name": "aind-data-upload-utils" +handlers: + console: + class: logging.StreamHandler + formatter: default + level: INFO + stream: ext://sys.stdout +loggers: + '': + handlers: + - console + level: INFO + propagate: true + uvicorn.error: + handlers: + - console + level: INFO + propagate: false + uvicorn.access: + handlers: + - console + level: INFO + propagate: false \ No newline at end of file