GenerateReport.yml

name: Generate Report

on:
  schedule:
    - cron: '0 0 * * MON,THU'
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - '**.py'
      - '.github/workflows/GenerateReport.yml'
      - '.github/workflows/PythonSetup/**'
      - '.github/workflows/RcloneSetup/**'
      - '.env'
      - 'src/**'
      - 'requirements.txt'
      - 'templates/**'
      - 'utils/**'

permissions:
  contents: write

jobs:
  healthcheck:
    name: Health Check
    runs-on: ubuntu-latest
    environment: WT_WeeklyTriggerEnv
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Call setup-python
        id: setup-python
        uses: ./.github/workflows/PythonSetup

      - name: Check pip health
        run: |
          pip --version
          python -c "import pip._vendor.resolvelib; print(pip._vendor.resolvelib.__file__)"

      - name: Print config.env
        run: grep -E '^(FULL_RELOAD_PACKAGES|BASE_PACKAGE_CSV|REQUIREMENTS_FILE)=' .env || true

      - name: Set up rclone
        uses: ./.github/workflows/RcloneSetup
        with:
          rclone_conf_base64: ${{ secrets.RCLONE_CONF_BASE64 }}

      - name: Check rclone connection
        run: rclone lsd gdrive:/Geek/PythonPackageManager/WeeklyReports/

  generate-report:
    name: Generate Report
    runs-on: ubuntu-latest
    needs: healthcheck
    environment: WT_WeeklyTriggerEnv
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Call setup-python
        id: setup-python
        uses: ./.github/workflows/PythonSetup

      - name: Install mutt (only on GitHub-hosted runners)
        if: contains(runner.labels, 'self-hosted') == false
        run: |
          sudo apt-get update
          sudo apt-get install -y mutt

      - name: Check pip health
        run: |
          pip --version
          python -c "import pip._vendor.resolvelib; print(pip._vendor.resolvelib.__file__)"

      - name: Print config.env
        run: grep -E '^(FULL_RELOAD_PACKAGES|BASE_PACKAGE_CSV|REQUIREMENTS_FILE)=' .env || true

      - name: Run report generation and extract personal report info
        id: run_report
        run: |
          output=$(python GenerateReport.py | tee /dev/stderr)

          summary_path="temp/PersonalReportSummary.txt"
          table_path="temp/PersonalReportEmail.html"
          echo "📄 Using fixed summary path: $summary_path"

          if [[ ! -f "$summary_path" ]]; then
            echo "⚠️ Summary file not found. No packages to upgrade."
            echo "UPGRADE_COUNT=0" >> $GITHUB_ENV
            echo "UPGRADE_PKG_LIST=" >> $GITHUB_ENV
            echo "UPGRADE_TABLE=" >> $GITHUB_ENV
          else
            count=$(grep -oP '^UPGRADE_COUNT=\K\d+' "$summary_path")
            pkgs=$(awk '/^PACKAGE_LIST:/ {flag=1; next} /^$/ {flag=0} flag' "$summary_path")
            echo "UPGRADE_COUNT=$count" >> $GITHUB_ENV
            echo "UPGRADE_PKG_LIST<<EOF" >> $GITHUB_ENV
            echo "$pkgs" >> $GITHUB_ENV
            echo "EOF" >> $GITHUB_ENV
            if [[ -f "$table_path" ]]; then
              echo "📄 Table file found: $table_path"
              echo "📏 Table file size: $(wc -c < "$table_path") bytes"
              echo "📋 Table file content preview:"
              head -5 "$table_path"
              echo "UPGRADE_TABLE<<EOF" >> $GITHUB_ENV
              cat "$table_path" >> $GITHUB_ENV
              echo "EOF" >> $GITHUB_ENV
            else
              echo "⚠️ Table file not found: $table_path"
              echo "UPGRADE_TABLE=" >> $GITHUB_ENV
            fi
          fi

      - name: Show changed files
        run: |
          git status
          git diff --name-only

      - name: Commit and push changes
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Set timezone to SGT (Singapore Time)
          export TZ="Asia/Singapore"
          SGT_TIME=$(date '+%Y-%m-%d %H:%M SGT')
          
          shopt -s globstar
          git add WeeklyReport/**/*.csv || true
          git commit -m "📝 Update WeeklyReport on $SGT_TIME" || echo "No changes in WeeklyReport"

          git add MonthlyReport/**/*.xlsx || true
          git commit -m "📊 Update MonthlyReport on $SGT_TIME" || echo "No changes in MonthlyReport"

          git add src/BasePackageWithDependencies.csv || true
          git commit -m "📦 Update BasePackageWithDependencies on $SGT_TIME" || echo "No changes in BasePackageWithDependencies"

          # Pull remote changes before pushing to avoid non-fast-forward errors
          git fetch origin ${{ github.ref_name }}
          
          # Try to rebase first, if it fails due to conflicts, try merge
          if ! git rebase origin/${{ github.ref_name }}; then
            echo "⚠️ Rebase failed, trying merge approach..."
            git reset --hard HEAD
            git pull origin ${{ github.ref_name }} --no-edit || {
              echo "⚠️ Pull failed, trying to force push our changes..."
              git push origin HEAD:${{ github.ref_name }} --force-with-lease || {
                echo "❌ All push attempts failed. Another workflow may have pushed changes."
                echo "ℹ️ This is expected behavior when multiple workflows run concurrently."
                echo "ℹ️ The workflow will continue with other tasks (releases, uploads, etc.)."
                exit 0  # Don't fail the workflow, just skip the push
              }
            }
          fi

          # Try to push, but don't fail the workflow if it doesn't work
          git push origin HEAD:${{ github.ref_name }} || {
            echo "⚠️ Push failed, but continuing with workflow..."
            echo "ℹ️ This may happen when other workflows push changes concurrently."
            echo "ℹ️ The workflow will continue with releases and uploads."
          }

      - name: Send Personal Report Email with mutt
        run: |
          subject=""
          body_file="email_body.txt"
      
          echo "UPGRADE_COUNT = ${{ env.UPGRADE_COUNT }}"
          echo "UPGRADE_TABLE length = ${#UPGRADE_TABLE}"
          echo "UPGRADE_TABLE preview = ${UPGRADE_TABLE:0:100}..."
          echo "Full UPGRADE_TABLE content:"
          echo "$UPGRADE_TABLE"
      
          if [ "${{ env.UPGRADE_COUNT }}" -eq 0 ]; then
            subject="✅ Personal Report - No packages need upgrade"
            printf 'Hello team,<br><br>No packages require upgrade this week.<br><br>Regards,<br>Report Bot\n' > "$body_file"
            attachments=""
          else
            subject="🔐 Personal Report - ${{ env.UPGRADE_COUNT }} packages need upgrade"
            printf 'Hello team,<br><br>🔧 Number of packages needing upgrade: %s<br><br>%s<br><br>Regards,<br>Report Bot\n' "${{ env.UPGRADE_COUNT }}" "${{ env.UPGRADE_TABLE }}" > "$body_file"
            attachments="-a temp/PersonalReport.csv -a temp/PersonalReport.html"
          fi
      
          echo "📧 Sending mail with subject: $subject"
      
          mutt -e "set smtp_url=smtp://${{ secrets.EMAIL_USERNAME }}:${{ secrets.EMAIL_PASSWORD }}@${{ secrets.SMTP_SERVER }}:${{ secrets.SMTP_PORT }}" \
               -e "set from='${{ secrets.EMAIL_USERNAME }}'" \
               -e "set content_type=text/html" \
               -s "$subject" $attachments -- "${{ secrets.EMAIL_TO_LIST }}" < "$body_file"
      
          if [ $? -eq 0 ]; then
            echo "✅ Email sent successfully."
          else
            echo "❌ Failed to send email. Please check the SMTP configuration and logs."
            exit 1
          fi



  upload-report:
    name: Upload to Google Drive
    runs-on: ubuntu-latest
    environment: WT_WeeklyTriggerEnv
    needs: generate-report
    steps:
      - name: Checkout latest commit from main
        run: |
          git init
          git remote add origin https://github.com/${{ github.repository }}
          git fetch origin main
          git checkout origin/main

      - name: Set up rclone
        uses: ./.github/workflows/RcloneSetup
        with:
          rclone_conf_base64: ${{ secrets.RCLONE_CONF_BASE64 }}

      - name: Check rclone connection
        run: rclone lsd gdrive:/Geek/PythonPackageManager/WeeklyReports/

      - name: Upload Weekly Report to Google Drive
        run: |
          set +e  # Allow script to continue even if rclone fails
      
          echo "📤 Start uploading via rclone..."
          OUTPUT=$(rclone copy WeeklyReport gdrive:/Geek/PythonPackageManager/WeeklyReports/ \
            --create-empty-src-dirs \
            --verbose \
            --checkers=8 \
            --stats=10s \
            --no-update-modtime \
            --drive-stop-on-upload-limit \
            2>&1)
          STATUS=$?
      
          echo "$OUTPUT"
      
          if [[ $STATUS -ne 0 ]]; then
            if echo "$OUTPUT" | grep -q "403: User rate limit exceeded"; then
              echo "⚠️ Rclone hit user rate limit (403), but continuing anyway."
              exit 0
            else
              echo "❌ Rclone failed with non-recoverable error."
              exit $STATUS
            fi
          else
            echo "✅ Rclone upload completed successfully."
          fi

      - name: Upload Monthly Report to Google Drive
        run: |
          set +e  # Allow script to continue even if rclone fails
      
          echo "📤 Start uploading via rclone..."
          OUTPUT=$(rclone copy MonthlyReport gdrive:/Geek/PythonPackageManager/MonthlyReports/ \
            --create-empty-src-dirs \
            --verbose \
            --checkers=8 \
            --stats=10s \
            --no-update-modtime \
            2>&1)
          STATUS=$?
      
          echo "$OUTPUT"
      
          if [[ $STATUS -ne 0 ]]; then
            if echo "$OUTPUT" | grep -q "403: User rate limit exceeded"; then
              echo "⚠️ Rclone hit user rate limit (403), but continuing anyway."
              exit 0
            else
              echo "❌ Rclone failed with non-recoverable error."
              exit $STATUS
            fi
          else
            echo "✅ Rclone upload completed successfully."
          fi

  create-monthly-release:
    name: Create Monthly Report Release
    runs-on: ubuntu-latest
    environment: WT_WeeklyTriggerEnv
    needs: generate-report
    if: success()
    steps:
      - name: Install GitHub CLI
        run: |
          sudo apt-get update
          sudo apt-get install -y gh jq

      - name: Checkout latest commit from main
        run: |
          git init
          git remote add origin https://github.com/${{ github.repository }}
          git fetch origin main
          git checkout origin/main

      - name: Find latest monthly report
        id: find_latest_monthly
        run: |
          echo "🔍 Scanning for latest MonthlyReport..."
          latest_file=$(find MonthlyReport -type f -name "MonthlyReport-*.xlsx" -printf '%T@ %p\n' | sort -n | tail -n 1 | cut -d' ' -f2-)

          if [[ -z "$latest_file" || ! -f "$latest_file" ]]; then
            echo "❌ No monthly report found."
            exit 1
          fi

          echo "✅ Found: $latest_file"

          filename=$(basename $latest_file)
          year_month=$(echo "$filename" | grep -oP '(?<=MonthlyReport-)[0-9]{6}')

          # Format month for display (e.g., "Jul 2025")
          year=$(echo "$year_month" | cut -c1-4)
          month_num=$(echo "$year_month" | cut -c5-6)
          month_name=$(date -d "${year}-${month_num}-01" +%b)
          month_display="${month_name} ${year}"

          # Get current date for "Generated at" part (SGT timezone)
          export TZ="Asia/Singapore"
          generated_date=$(date '+%Y-%m-%d %H:%M:%S SGT')

          echo "📦 Scanning existing monthly tags with prefix: monthly-${year_month}-Ver"

          existing_tags=$(gh release list --limit 100 --json tagName | jq -r ".[] | .tagName" | grep -E "^monthly-${year_month}-Ver[0-9]+$" || true)

          max_ver=$(echo "$existing_tags" | grep -oE 'Ver[0-9]+' | sed 's/Ver//' | sort -n | tail -n 1)

          if [[ -z "$max_ver" ]]; then
            new_ver=1
          else
            new_ver=$((max_ver + 1))
          fi

          tag="monthly-${year_month}-Ver${new_ver}"
          echo "🆕 New tag: $tag"

          echo "RELEASE_TAG=$tag" >> $GITHUB_ENV
          echo "REPORT_FILE=$latest_file" >> $GITHUB_ENV
          echo "REPORT_NAME=$filename" >> $GITHUB_ENV
          echo "MONTH_DISPLAY=$month_display" >> $GITHUB_ENV
          echo "GENERATED_DATE=$generated_date" >> $GITHUB_ENV
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Create GitHub Release
        run: |
          gh release create "$RELEASE_TAG" \
            "$REPORT_FILE#${REPORT_NAME}" \
            --title "Monthly Report - $MONTH_DISPLAY (Generated at $GENERATED_DATE)" \
            --notes "Automated monthly report"
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Remove old releases for the same month (keep only latest)
        run: |
          echo "🧹 Cleaning up other releases for the same month..."

          current_month_prefix=$(echo "${{ env.RELEASE_TAG }}" | grep -oP 'monthly-\K[0-9]{6}')

          # Get all releases for the same month, sorted by creation date
          releases=$(gh release list --limit 100 --json tagName,createdAt | \
            jq -c '.[] | select(.tagName | test("^monthly-'$current_month_prefix'-Ver[0-9]+$")) | {tag: .tagName, created: .createdAt}' | \
            jq -s 'sort_by(.created) | reverse')

          # Keep only the latest release (first in the sorted list)
          latest_tag=$(echo "$releases" | jq -r '.[0].tag')
          
          echo "✅ Keeping latest release: $latest_tag"

          # Delete all other releases for the same month
          echo "$releases" | jq -c '.[]' | while read line; do
            tag=$(echo "$line" | jq -r '.tag')

            if [[ "$tag" == "$latest_tag" ]]; then
              echo "⏭️ Skipping latest release: $tag"
              continue
            fi

            echo "🗑️ Deleting other release from the same month: $tag"
            gh release delete "$tag" --yes || echo "⚠️ Failed to delete release $tag"
            if git ls-remote --tags origin | grep -q "refs/tags/$tag$"; then
              echo "🔖 Tag $tag exists. Deleting..."
              gh api -X DELETE "repos/${{ github.repository }}/git/refs/tags/$tag" || echo "⚠️ Failed to delete tag ref $tag"
            else
              echo "ℹ️ Tag $tag does not exist. Skipping tag deletion."
            fi 
          done
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

  create-weekly-release:
    name: Create Weekly Report Release
    runs-on: ubuntu-latest
    environment: WT_WeeklyTriggerEnv
    needs: create-monthly-release
    if: success()

    steps:
      - name: Install GitHub CLI
        run: |
          sudo apt-get update
          sudo apt-get install -y gh

      - name: Checkout latest commit from main
        run: |
          git init
          git remote add origin https://github.com/${{ github.repository }}
          git fetch origin main
          git checkout origin/main

      - name: Find latest report set
        id: find_latest
        run: |
          echo "📁 Scanning for latest report set..."

          latest_csv=$(find WeeklyReport -type f -name "WeeklyReport_*.csv" -printf '%T@ %p\n' | sort -n | tail -n 1 | cut -d' ' -f2-)

          if [[ -z "$latest_csv" || ! -f "$latest_csv" ]]; then
            echo "❌ No weekly CSV report found."
            exit 1
          fi

          base_name="${latest_csv%.csv}"
          html_file="${base_name}.html"
          json_file="${base_name}.json"

          echo "📝 Latest base: $base_name"
          echo "📄 CSV:  $latest_csv"
          echo "📄 HTML: $html_file"
          echo "📄 JSON: $json_file"

          echo "CSV_PATH=$latest_csv" >> $GITHUB_ENV
          echo "CSV_FILENAME=$(basename $latest_csv)" >> $GITHUB_ENV

          if [[ -f "$html_file" ]]; then
            echo "HTML_PATH=$html_file" >> $GITHUB_ENV
            echo "HTML_FILENAME=$(basename $html_file)" >> $GITHUB_ENV
          else
            echo "⚠️ HTML report not found: $html_file"
            echo "HTML_PATH=" >> $GITHUB_ENV
            echo "HTML_FILENAME=" >> $GITHUB_ENV
          fi

          if [[ -f "$json_file" ]]; then
            echo "JSON_PATH=$json_file" >> $GITHUB_ENV
            echo "JSON_FILENAME=$(basename $json_file)" >> $GITHUB_ENV
          else
            echo "⚠️ JSON report not found: $json_file"
            echo "JSON_PATH=" >> $GITHUB_ENV
            echo "JSON_FILENAME=" >> $GITHUB_ENV
          fi

          # 🗓️ Get the Monday of current week
          MONDAY_DATE=$(date -d "$(date +%Y-%m-%d -d @$(( $(date +%s) - ($(date +%u) - 1) * 86400 )))" +%Y-%m-%d)
          echo "📅 This week's Monday: $MONDAY_DATE"

          # Get current date for "Generated at" part (SGT timezone)
          export TZ="Asia/Singapore"
          generated_date=$(date '+%Y-%m-%d %H:%M:%S SGT')

          # 🔍 Find existing tags in this week and determine Ver{x}
          existing_tags=$(gh release list --limit 100 --json tagName | jq -r ".[] | .tagName" | grep -E "^weekly-${MONDAY_DATE//-}-Ver[0-9]+$" || true)
          max_ver=$(echo "$existing_tags" | grep -oE 'Ver[0-9]+' | sed 's/Ver//' | sort -n | tail -n 1)

          if [[ -z "$max_ver" ]]; then
            new_ver=1
          else
            new_ver=$((max_ver + 1))
          fi

          tag="weekly-${MONDAY_DATE//-}-Ver${new_ver}"
          echo "🆕 Release tag: $tag"

          echo "RELEASE_TAG=$tag" >> $GITHUB_ENV
          echo "MONDAY_DATE_DISPLAY=$MONDAY_DATE" >> $GITHUB_ENV
          echo "GENERATED_DATE=$generated_date" >> $GITHUB_ENV
          echo "release_tag=$tag" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Create GitHub Release
        run: |
          gh release create "$RELEASE_TAG" \
            "$CSV_PATH#${CSV_FILENAME}" \
            --title "Weekly Report - $MONDAY_DATE_DISPLAY (Generated at $GENERATED_DATE)" \
            --notes "Automated weekly report"

          if [[ -n "$HTML_PATH" ]]; then
            gh release upload "$RELEASE_TAG" "$HTML_PATH#${HTML_FILENAME}" --clobber
          fi

          if [[ -n "$JSON_PATH" ]]; then
            gh release upload "$RELEASE_TAG" "$JSON_PATH#${JSON_FILENAME}" --clobber
          fi
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Remove old releases for the same week (keep only latest)
        run: |
          echo "🧹 Cleaning up other releases for the same week..."

          current_week_prefix=$(echo "${{ env.RELEASE_TAG }}" | grep -oP 'weekly-\K[0-9]{8}')

          # Get all releases for the same week, sorted by creation date
          releases=$(gh release list --limit 100 --json tagName,createdAt | \
            jq -c '.[] | select(.tagName | test("^weekly-'$current_week_prefix'-Ver[0-9]+$")) | {tag: .tagName, created: .createdAt}' | \
            jq -s 'sort_by(.created) | reverse')

          # Keep only the latest release (first in the sorted list)
          latest_tag=$(echo "$releases" | jq -r '.[0].tag')
          
          echo "✅ Keeping latest release: $latest_tag"

          # Delete all other releases for the same week
          echo "$releases" | jq -c '.[]' | while read line; do
            tag=$(echo "$line" | jq -r '.tag')

            if [[ "$tag" == "$latest_tag" ]]; then
              echo "⏭️ Skipping latest release: $tag"
              continue
            fi

            echo "🗑️ Deleting other release from the same week: $tag"
            gh release delete "$tag" --yes || echo "⚠️ Failed to delete release $tag"
            if git ls-remote --tags origin | grep -q "refs/tags/$tag$"; then
              echo "🔖 Tag $tag exists. Deleting..."
              gh api -X DELETE "repos/${{ github.repository }}/git/refs/tags/$tag" || echo "⚠️ Failed to delete tag ref $tag"
            else
              echo "ℹ️ Tag $tag does not exist. Skipping tag deletion."
            fi
          done
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

  sync-main-to-dev:
    name: Sync main to dev
    needs: create-weekly-release
    uses: ./.github/workflows/SyncMainToDev.yml
    permissions:
      contents: write
    secrets: inherit