analyze_log_frequency.py

#!/usr/bin/env python3
"""
Log Line Frequency Analyzer

Reads a log file, counts occurrences of identical lines,
and outputs results sorted by frequency (most to least frequent).
"""

import sys
from collections import Counter
import argparse


def analyze_log_file(input_file, output_file):
    """Analyze log file and write frequency results."""
    try:
        # Read all lines from the input file
        with open(input_file, 'r', encoding='utf-8', errors='ignore') as f:
            lines = [line.rstrip('\n\r') for line in f]

        # Count occurrences of each line
        line_counts = Counter(lines)

        # Sort by frequency (descending) then by line content for consistency
        sorted_lines = sorted(line_counts.items(), key=lambda x: (-x[1], x[0]))

        # Write results to output file
        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(f"Log File Frequency Analysis: {input_file}\n")
            f.write("=" * 60 + "\n")
            f.write(f"Total unique lines: {len(sorted_lines)}\n")
            f.write(f"Total lines processed: {len(lines)}\n")
            f.write("=" * 60 + "\n\n")

            for line_content, count in sorted_lines:
                f.write(f"{count:6d} | {line_content}\n")

        print(f"Analysis complete. Results written to: {output_file}")
        print(f"Processed {len(lines)} total lines, {len(sorted_lines)} unique lines")

        # Show top 10 most frequent lines
        print("\nTop 10 most frequent lines:")
        for i, (line_content, count) in enumerate(sorted_lines[:10]):
            preview = line_content[:80] + "..." if len(line_content) > 80 else line_content
            print(f"{i+1:2d}. {count:6d} times: {preview}")

    except FileNotFoundError:
        print(f"Error: File '{input_file}' not found.")
        sys.exit(1)
    except Exception as e:
        print(f"Error processing file: {e}")
        sys.exit(1)


def main():
    parser = argparse.ArgumentParser(
        description="Analyze log file line frequencies",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s logfile.txt                    # Output to logfile_frequency.txt
  %(prog)s logfile.txt -o results.txt     # Output to results.txt
  %(prog)s ~/.local/share/nvim/diffusion.log  # Analyze diffusion log
        """
    )

    parser.add_argument('input_file', help='Input log file to analyze')
    parser.add_argument('-o', '--output', help='Output file (default: <input>_frequency.txt)')

    args = parser.parse_args()

    # Generate output filename if not provided
    if args.output:
        output_file = args.output
    else:
        if args.input_file.endswith('.log'):
            output_file = args.input_file.replace('.log', '_frequency.txt')
        else:
            output_file = args.input_file + '_frequency.txt'

    analyze_log_file(args.input_file, output_file)


if __name__ == '__main__':
    main()