From 0c9d7d0e8ec400951bde7e9e813fe7892c71d8f2 Mon Sep 17 00:00:00 2001 From: brott8 Date: Wed, 15 May 2024 16:19:20 +1000 Subject: [PATCH 1/3] - Integration of Apprise notification services as a configurable option for Snapraid process results. - Update of touch regex to account for "a" in versions of Snapraid - Handling split parity in sanity checks --- README.md | 2 +- config.json.example | 5 ++ config.schema.json | 34 +++++++-- reports/apprise_report.py | 148 ++++++++++++++++++++++++++++++++++++++ snapper.py | 42 +++++++++-- 5 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 reports/apprise_report.py diff --git a/README.md b/README.md index d40dd71..1c2c504 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The reason I created this is that I wanted more granular control of how my setup - Allows you to abort execution if configurable thresholds are broken - Allows you to `scrub` after `sync` - Logs the raw snapraid output as well as formatted text -- Creates a nicely formatted report and sends it via email or discord +- Creates a nicely formatted report and sends it via email, Discord, or [Apprise](https://github.com/caronc/apprise) - Provides live insight into the sync/scrub process in Discord - Spin down selected hard drives after script completion diff --git a/config.json.example b/config.json.example index ee9483a..5efdd4a 100644 --- a/config.json.example +++ b/config.json.example @@ -34,6 +34,11 @@ "enabled": false, "webhook_id": "", "webhook_token": "" + }, + "apprise": { + "enabled": false, + "binary": "/usr/bin/apprise", + "config": "/var/lib/apprise/config.yml" } }, "logs": { diff --git a/config.schema.json b/config.schema.json index 5c0ee7e..57c9709 100644 --- a/config.schema.json +++ b/config.schema.json @@ -230,15 +230,37 @@ "required": [ "enabled", "webhook_id", - "webhook_token" - ] + "webhook_token"] + }, + "apprise": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "examples": [ + true + ], + "description": "Whether or not to send notifications and reports to Apprise." + }, + "binary": { + "type": "string", + "examples": [ + "/usr/bin/apprise" + ], + "description": "The location of `apprise`." + }, + "config": { + "type": "string", + "examples": ["/etc/apprise.yml"], + "description": "Location of the apprise configuration yml." + } + }, + "additionalProperties": false, + "required": ["enabled", "binary", "config"] } }, "additionalProperties": false, - "required": [ - "email", - "discord" - ] + "required": ["email", "discord", "apprise"] }, "logs": { "type": "object", diff --git a/reports/apprise_report.py b/reports/apprise_report.py new file mode 100644 index 0000000..3ac15fa --- /dev/null +++ b/reports/apprise_report.py @@ -0,0 +1,148 @@ +from operator import itemgetter + + +def create_apprise_report(report_data): + sync_job_ran, scrub_job_ran, sync_job_time, scrub_job_time, diff_data, zero_subsecond_count, \ + scrub_stats, drive_stats, smart_drive_data, global_fp, total_time = itemgetter( + 'sync_job_ran', + 'scrub_job_ran', + 'sync_job_time', + 'scrub_job_time', + 'diff_data', + 'zero_subsecond_count', + 'scrub_stats', + 'drive_stats', + 'smart_drive_data', + 'global_fp', + 'total_time')(report_data) + + # + # Create email report + + sync_report = f'**Sync Job**' + + if sync_job_ran: + sync_report = sync_report + f''' + Job finished successfully in {sync_job_time}. + File diff summary as follows: + - {diff_data["added"]} added + - {diff_data["removed"]} removed + - {diff_data["updated"]} updated + - {diff_data["moved"]} moved + - {diff_data["copied"]} copied + - {diff_data["restored"]} restored + ''' + else: + sync_report = sync_report + 'Sync job did **not** run.' + + touch_report = '**Touch job**' + + if zero_subsecond_count > 0: + touch_report = touch_report + f''' + 'A total of {zero_subsecond_count} file(s) had their sub-second value fixed. + ''' + else: + touch_report = touch_report + 'No zero sub-second files were found.' + + scrub_report = '**Scrub Job**' + + if scrub_job_ran: + scrub_report = scrub_report + f''' + Job finished successfully in {scrub_job_time}. + {scrub_stats["unscrubbed"]}% of the array has not been scrubbed, with the oldest block at {scrub_stats["scrub_age"]} day(s). + ''' + # , the median at {scrub_stats["median"]} day(s), and the newest at {scrub_stats["newest"]} day(s). + else: + scrub_report = scrub_report + 'Scrub Job did **not** run.' + + array_drive_report = ''.join(f''' + + {d["drive_name"] if d["drive_name"] else 'Full Array'} + {d["fragmented_files"]} + {d["excess_fragments"]} + {d["wasted_gb"]} + {d["used_gb"]} + {d["free_gb"]} + {d["use_percent"]} + + ''' for d in drive_stats) + + array_report = f''' +

SnapRAID Array Report

+ + + + + + + + + + + + + + {array_drive_report} + +
DriveFragmented FilesExcess FragmentsWasted Space (GB)Used Space (GB)Free Space (GB)Total Used (%)
+ ''' + + smart_drive_report = ''.join(f''' + + {d["disk"]} ({d["device"]}) + {d["temp"]} + {d["power_on_days"]} + {d["error_count"]} + {d["fp"]} + {d["size"]} + {d["serial"]} + + ''' for d in smart_drive_data) + + smart_report = f''' +

SMART Report

+ + + + + + + + + + + + + + {smart_drive_report} + +
DriveTemperature (°C)Power On Time (days)Error CountFailure ProbabilityDrive Size (TiB)Serial Number
+

The current failure probability of any single drive this year is {global_fp}%.

+ ''' + + # Check if any drive had an error count + drives_with_errors = [] + for drive in smart_drive_data: + error_count = drive.get("error_count") + if isinstance(error_count, str) and error_count.isdigit(): + error_count = int(error_count) + if isinstance(error_count, int) and error_count > 0: + drive["error_count"] = error_count + drives_with_errors.append(drive) + + # Summarize SMART drive report + smart_summary = f'**SMART Summary**\nFailure Probability: {global_fp}%' + + if drives_with_errors: + smart_summary += "\nDrives with errors:" + for drive in drives_with_errors: + smart_summary += f"\n- {drive['disk']} ({drive['device']}) - Error Count: {drive['error_count']}" + + email_report = f'''SnapRAID job completed successfully in {total_time} + {touch_report} + {sync_report} + {scrub_report} + {smart_summary} + ''' + + return email_report diff --git a/snapper.py b/snapper.py index dd7f3f2..fd23ad6 100644 --- a/snapper.py +++ b/snapper.py @@ -20,6 +20,7 @@ from reports.discord_report import create_discord_report from reports.email_report import create_email_report +from reports.apprise_report import create_apprise_report from utils import format_delta, get_relative_path, human_readable_size, run_script # @@ -101,7 +102,8 @@ def notify_and_handle_error(message, error): def notify_warning(message, embeds=None): - return send_discord(f':warning: [**WARNING!**] {message}', embeds=embeds) + send_apprise(':warning: [**WARNING!**]', message) + send_discord(f':warning: [**WARNING!**] {message}', embeds=embeds) def notify_info(message, embeds=None, message_id=None): @@ -179,6 +181,32 @@ def send_email(subject, message): log.debug(f'Successfully sent email to {to_email}') +def send_apprise(subject, message): + log.debug('Attempting to send apprise notification...') + + is_enabled, apprise_bin, config_loc = itemgetter( + 'enabled', 'binary', 'config')(config['notifications']['apprise']) + + if not is_enabled: + return + + if not os.path.isfile(apprise_bin): + raise FileNotFoundError('Unable to find apprise executable', apprise_bin) + + result = subprocess.run([ + apprise_bin, + '-vv', + '-t', subject, + '-b', message, + '--config=' + config_loc + ], capture_output=True, text=True) + + if result.stderr: + raise ConnectionError('Unable to send notification', result.stderr) + + log.debug(f'Successfully sent apprise notification') + + # # Snapraid Helpers @@ -319,7 +347,7 @@ def get_status(): error_count = re.search(r'^DANGER! In the array there are (?P\d+) errors!', snapraid_status, flags=re.MULTILINE) zero_subsecond_count = re.search( - r'^You have (?P\d+) files with (?:a )?zero sub-second timestamp', snapraid_status, + r'^You have (?P\d+) files with( a)? zero sub-second timestamp', snapraid_status, flags=re.MULTILINE) sync_in_progress = bool( @@ -535,7 +563,8 @@ def get_snapraid_config(): with open(config_file, 'r') as file: snapraid_config = file.read() - file_regex = re.compile(r'^(content|(?:\d+-)?parity) +(.+/\w+.(?:content|(?:\d+-)?parity)) *$', + #Split parity handling + file_regex = re.compile(r'^(content|parity) +(.+/\w+.(?:content|parity)) *$', flags=re.MULTILINE) parity_files = [] content_files = [] @@ -544,7 +573,8 @@ def get_snapraid_config(): if m[1] == 'content': content_files.append(m[2]) else: - parity_files.append(m[2]) + for p in m[2].split(','): + parity_files.append(p) return content_files, parity_files @@ -679,9 +709,11 @@ def main(): } email_report = create_email_report(report_data) - send_email('SnapRAID Job Completed Successfully', email_report) + apprise_report = create_apprise_report(report_data) + send_apprise('SnapRAID Job Completed Successfully', apprise_report) + if config['notifications']['discord']['enabled']: (discord_message, embeds) = create_discord_report(report_data) From 8eeb4ed94dc36f5f5b3757eeb7048e640a0b5357 Mon Sep 17 00:00:00 2001 From: brott8 Date: Sun, 1 Sep 2024 13:24:52 +1000 Subject: [PATCH 2/3] - Added guarding config check for send_apprise and send_email - Reverting touch regex - Reverting regex for split parity --- snapper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/snapper.py b/snapper.py index fd23ad6..5561d30 100644 --- a/snapper.py +++ b/snapper.py @@ -347,7 +347,7 @@ def get_status(): error_count = re.search(r'^DANGER! In the array there are (?P\d+) errors!', snapraid_status, flags=re.MULTILINE) zero_subsecond_count = re.search( - r'^You have (?P\d+) files with( a)? zero sub-second timestamp', snapraid_status, + r'^You have (?P\d+) files with (?:a )?zero sub-second timestamp', snapraid_status, flags=re.MULTILINE) sync_in_progress = bool( @@ -564,7 +564,7 @@ def get_snapraid_config(): snapraid_config = file.read() #Split parity handling - file_regex = re.compile(r'^(content|parity) +(.+/\w+.(?:content|parity)) *$', + file_regex = re.compile(r'^(content|(?:\d+-)?parity) +(.+/\w+.(?:content|(?:\d+-)?parity)) *$', flags=re.MULTILINE) parity_files = [] content_files = [] @@ -708,11 +708,13 @@ def main(): 'total_time': total_time } - email_report = create_email_report(report_data) - send_email('SnapRAID Job Completed Successfully', email_report) + if config['notifications']['email']['enabled']: + email_report = create_email_report(report_data) + send_email('SnapRAID Job Completed Successfully', email_report) - apprise_report = create_apprise_report(report_data) - send_apprise('SnapRAID Job Completed Successfully', apprise_report) + if config['notifications']['apprise']['enabled']: + apprise_report = create_apprise_report(report_data) + send_apprise('SnapRAID Job Completed Successfully', apprise_report) if config['notifications']['discord']['enabled']: (discord_message, embeds) = create_discord_report(report_data) From 53ec2b80b3d19076004fc84919e8c858bc76b8a6 Mon Sep 17 00:00:00 2001 From: brott8 Date: Sun, 1 Sep 2024 13:28:03 +1000 Subject: [PATCH 3/3] - Removed extraneous array report html from apprise report --- reports/apprise_report.py | 65 --------------------------------------- 1 file changed, 65 deletions(-) diff --git a/reports/apprise_report.py b/reports/apprise_report.py index 3ac15fa..6b2574d 100644 --- a/reports/apprise_report.py +++ b/reports/apprise_report.py @@ -55,71 +55,6 @@ def create_apprise_report(report_data): else: scrub_report = scrub_report + 'Scrub Job did **not** run.' - array_drive_report = ''.join(f''' - - {d["drive_name"] if d["drive_name"] else 'Full Array'} - {d["fragmented_files"]} - {d["excess_fragments"]} - {d["wasted_gb"]} - {d["used_gb"]} - {d["free_gb"]} - {d["use_percent"]} - - ''' for d in drive_stats) - - array_report = f''' -

SnapRAID Array Report

- - - - - - - - - - - - - - {array_drive_report} - -
DriveFragmented FilesExcess FragmentsWasted Space (GB)Used Space (GB)Free Space (GB)Total Used (%)
- ''' - - smart_drive_report = ''.join(f''' - - {d["disk"]} ({d["device"]}) - {d["temp"]} - {d["power_on_days"]} - {d["error_count"]} - {d["fp"]} - {d["size"]} - {d["serial"]} - - ''' for d in smart_drive_data) - - smart_report = f''' -

SMART Report

- - - - - - - - - - - - - - {smart_drive_report} - -
DriveTemperature (°C)Power On Time (days)Error CountFailure ProbabilityDrive Size (TiB)Serial Number
-

The current failure probability of any single drive this year is {global_fp}%.

- ''' - # Check if any drive had an error count drives_with_errors = [] for drive in smart_drive_data: