diff --git a/log_config_template.yaml b/log_config_template.yaml new file mode 100644 index 0000000..b24e24a --- /dev/null +++ b/log_config_template.yaml @@ -0,0 +1,61 @@ +--- +version: 1 +disable_existing_loggers: false +formatters: + default: + "()": aind_data_upload_utils.CustomJsonFormatter + format: "%(asctime)s %(levelname)s %(module)s %(lineno)s %(message)s" + reserved_attrs: + - args + - asctime + - created + - exc_info + - exc_text + - filename + - funcName + - levelname + - levelno + - lineno + - message + - module + - msecs + - msg + - name + - pathname + - process + - processName + - relativeCreated + - stack_info + - taskName + - thread + - threadName + - color_message + rename_fields: + "asctime": "timestamp" + "levelname": "level" + static_fields: + "environment": "local" + "software_version": ext://aind_data_upload_utils.__version__ + "software_name": "aind-data-upload-utils" +handlers: + console: + class: logging.StreamHandler + formatter: default + level: INFO + stream: ext://sys.stdout +loggers: + '': + handlers: + - console + level: INFO + propagate: true + uvicorn.error: + handlers: + - console + level: INFO + propagate: false + uvicorn.access: + handlers: + - console + level: INFO + propagate: false \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a66e54e..cec1eba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,9 @@ dependencies = [ 'pydantic>2.9', 'pydantic-settings>=2.8', 'boto3', - 'requests' + 'requests', + 'python-json-logger', + 'PyYAML' ] [project.optional-dependencies] diff --git a/src/aind_data_upload_utils/__init__.py b/src/aind_data_upload_utils/__init__.py index 63ce3dd..e38b640 100644 --- a/src/aind_data_upload_utils/__init__.py +++ b/src/aind_data_upload_utils/__init__.py @@ -1,3 +1,43 @@ """Init package""" +import logging.config +import os +from datetime import datetime, timezone +from logging import LogRecord + +import yaml +from pythonjsonlogger import json as log_json + __version__ = "0.18.2" + + +# We want to standardize the timestamp format to UTC and ISO-8601, which +# requires a custom formatter and can't be done through configuration only. +class CustomJsonFormatter(log_json.JsonFormatter): + """Custom class to format log timestamps as ISO-8601 UTC""" + + def formatTime(self, record: LogRecord, datefmt=None) -> str: + """ + Format timestamp as ISO-8601 UTC + + Parameters + ---------- + record : LogRecord + datefmt : str, optional + Default is None. Unused parameter, kept for signature compatibility. + + Returns + ------- + str + + """ + dt = datetime.fromtimestamp(record.created, tz=timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +if os.path.isfile(os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml")): + config_path = os.getenv("LOGGING_CONFIG_FILE", "log_config.yaml") + with open(config_path, "rt") as f: + config = yaml.safe_load(f.read()) + logging.config.dictConfig(config) + logging.info(f"Found logging file at: {config_path}") diff --git a/src/aind_data_upload_utils/check_directories_job.py b/src/aind_data_upload_utils/check_directories_job.py index d2f086d..8688b01 100644 --- a/src/aind_data_upload_utils/check_directories_job.py +++ b/src/aind_data_upload_utils/check_directories_job.py @@ -16,10 +16,6 @@ from pydantic import BaseModel, Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class DirectoriesToCheckConfigs(BaseModel): """Basic model needed from BasicUploadConfigs""" @@ -206,12 +202,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/check_metadata_job.py b/src/aind_data_upload_utils/check_metadata_job.py index 05a466e..4cfc986 100644 --- a/src/aind_data_upload_utils/check_metadata_job.py +++ b/src/aind_data_upload_utils/check_metadata_job.py @@ -4,8 +4,6 @@ import argparse import json -import logging -import os import sys from pathlib import Path from typing import Set, Tuple, Union @@ -13,10 +11,6 @@ from pydantic import Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings(BaseSettings): """Job settings for CheckMetadataJob""" @@ -117,12 +111,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/copy_metadata_files.py b/src/aind_data_upload_utils/copy_metadata_files.py index 56795cd..330cd66 100644 --- a/src/aind_data_upload_utils/copy_metadata_files.py +++ b/src/aind_data_upload_utils/copy_metadata_files.py @@ -4,7 +4,6 @@ import argparse import logging -import os import shutil import sys from pathlib import Path @@ -13,10 +12,6 @@ from pydantic import Field from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings(BaseSettings): """Job settings for CheckMetadataJob""" @@ -81,12 +76,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/create_s5_commands_job.py b/src/aind_data_upload_utils/create_s5_commands_job.py index 3ba85a1..aedd550 100644 --- a/src/aind_data_upload_utils/create_s5_commands_job.py +++ b/src/aind_data_upload_utils/create_s5_commands_job.py @@ -18,10 +18,6 @@ from pydantic import Field, model_validator from pydantic_settings import BaseSettings -# Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings(BaseSettings): """Job settings for CreateS5CommandsJob""" @@ -221,12 +217,10 @@ def run_job(self) -> None: "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/create_sym_links_job.py b/src/aind_data_upload_utils/create_sym_links_job.py index bdd1cb7..64443a8 100644 --- a/src/aind_data_upload_utils/create_sym_links_job.py +++ b/src/aind_data_upload_utils/create_sym_links_job.py @@ -9,9 +9,6 @@ from pydantic import Field from pydantic_settings import BaseSettings -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings( BaseSettings, cli_parse_args=True, cli_ignore_unknown_args=True diff --git a/src/aind_data_upload_utils/delete_folders_job.py b/src/aind_data_upload_utils/delete_folders_job.py index 84e463e..d2a992a 100644 --- a/src/aind_data_upload_utils/delete_folders_job.py +++ b/src/aind_data_upload_utils/delete_folders_job.py @@ -4,7 +4,6 @@ import argparse import logging -import os import re import sys from pathlib import Path @@ -19,9 +18,6 @@ DeleteStagingFolderJob, ) -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings(BaseSettings): """Job settings for DeleteFoldersJob""" @@ -88,12 +84,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_source_folders_job.py b/src/aind_data_upload_utils/delete_source_folders_job.py index aacbbaf..55f141f 100644 --- a/src/aind_data_upload_utils/delete_source_folders_job.py +++ b/src/aind_data_upload_utils/delete_source_folders_job.py @@ -21,9 +21,6 @@ DeleteStagingFolderJob, ) -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) - class DirectoriesToDeleteConfigs(BaseModel): """Basic model that can be easily passed via the transfer service.""" @@ -232,12 +229,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_staging_folder_job.py b/src/aind_data_upload_utils/delete_staging_folder_job.py index b2e7634..3c0910e 100644 --- a/src/aind_data_upload_utils/delete_staging_folder_job.py +++ b/src/aind_data_upload_utils/delete_staging_folder_job.py @@ -17,8 +17,6 @@ from pydantic_settings import BaseSettings # Set log level from env var -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -logging.basicConfig(level=LOG_LEVEL) class JobSettings(BaseSettings): @@ -191,12 +189,10 @@ def run_job(self): "--job-settings", required=False, type=str, - help=( - r""" + help=(r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """ - ), + """), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py index f983bd3..9b17e00 100644 --- a/src/aind_data_upload_utils/trigger_co_cleanup_notification.py +++ b/src/aind_data_upload_utils/trigger_co_cleanup_notification.py @@ -5,7 +5,6 @@ import argparse import csv import logging -import os import sys from collections import defaultdict from io import StringIO @@ -17,9 +16,6 @@ from pydantic import Field from pydantic_settings import BaseSettings -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") -logging.basicConfig(level=LOG_LEVEL) - class JobSettings(BaseSettings): """Job settings for WebhookNotificationJob"""