diff --git a/.github/workflows/fetch_ingest_proofs.yml b/.github/workflows/fetch_ingest_proofs.yml new file mode 100644 index 0000000..91a6f04 --- /dev/null +++ b/.github/workflows/fetch_ingest_proofs.yml @@ -0,0 +1,88 @@ +name: Fetch and add new proofs + +on: + workflow_dispatch: + schedule: + # once a week + - cron: '0 0 * * 0' + +permissions: + contents: write + pull-requests: write + +jobs: + fetch-and-add: + env: + BRANCH: proof-auto-add-${{ github.run_id }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + TOP_SOURCES_LIST: ${{ vars.TOP_SOURCES_LIST }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + API_KEY: ${{ secrets.API_KEY }} + API_BASE_URL: ${{ vars.API_BASE_URL }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Setup Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create branch from main + run: | + git checkout -b "$BRANCH" origin/main + + - name: Run proof fetch script (with retries) + run: | + set -e + attempt=1 + max=3 + until [ $attempt -gt $max ]; do + echo "Attempt $attempt..." + if python scripts/ingest_proofs.py; then + echo "Script succeeded" + break + fi + attempt=$((attempt+1)) + if [ $attempt -le $max ]; then + echo "Retrying in 60 seconds..." + sleep 60 + fi + done + if [ $attempt -gt $max ]; then + echo "Script failed after $max attempts" + exit 1 + fi + + - name: Commit and push changes + id: commit + run: | + git add --all + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No changes to commit" + else + git commit -m "chore(proofs): adds new proofs for the ingested claims" + git push origin "$BRANCH" + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Create Pull Request + if: steps.commit.outputs.changed == 'true' + run: | + gh pr create \ + --title "Auto PR: Scheduled addition of new proofs" \ + --body "Adds proofs for claims made by the ingested sources in the last week for ingestion" \ + --base main diff --git a/.github/workflows/post_on_merge.yml b/.github/workflows/post_on_merge.yml index caa9009..e63f260 100644 --- a/.github/workflows/post_on_merge.yml +++ b/.github/workflows/post_on_merge.yml @@ -26,13 +26,12 @@ jobs: - name: Get added files in latest commit id: added run: | - files=$(git diff --diff-filter=A HEAD~1 HEAD --name-only | grep -E '^(sources|claims|proofs)/' || true) + files=$(git diff --diff-filter=A HEAD~1 HEAD --name-only | grep -E '^(sources|claims)/' || true) echo "Added files:" echo "$files" - # Order: sources, claims, proofs + # Order: sources, claims ordered_files=$(echo "$files" | grep '^sources/' || true) ordered_files+=$'\n'$(echo "$files" | grep '^claims/' || true) - ordered_files+=$'\n'$(echo "$files" | grep '^proofs/' || true) # Trim only leading/trailing whitespace, preserve internal spaces in filenames ordered_files=$(echo "$ordered_files" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') echo "ADDED_FILES<> "$GITHUB_ENV" diff --git a/.github/workflows/post_proofs_on_merge.yml b/.github/workflows/post_proofs_on_merge.yml new file mode 100644 index 0000000..ff16283 --- /dev/null +++ b/.github/workflows/post_proofs_on_merge.yml @@ -0,0 +1,47 @@ +name: POST New Proof Documents on Merge + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + post: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Get added files in latest commit + id: added + run: | + files=$(git diff --diff-filter=A HEAD~1 HEAD --name-only | grep -E '^(proofs)/' || true) + echo "Added files:" + echo "$files" + # Trim only leading/trailing whitespace, preserve internal spaces in filenames + files=$(echo "$files" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + echo "ADDED_FILES<> "$GITHUB_ENV" + echo "$files" >> "$GITHUB_ENV" + echo "EOF" >> "$GITHUB_ENV" + + - name: POST new documents + if: env.ADDED_FILES != '' + run: python scripts/post_requests.py + env: + API_BASE_URL: ${{ vars.API_BASE_URL }} + API_KEY: ${{ secrets.API_KEY }} + + - name: Nothing to POST + if: env.ADDED_FILES == '' + run: echo "No new request documents to POST." \ No newline at end of file diff --git a/.github/workflows/refresh_scores.yml b/.github/workflows/refresh_scores.yml index ca16df8..13df0be 100644 --- a/.github/workflows/refresh_scores.yml +++ b/.github/workflows/refresh_scores.yml @@ -2,7 +2,7 @@ name: Verify Claims and Source Scores on: workflow_run: - workflows: ["POST New Documents on Merge"] + workflows: ["POST New Proof Documents on Merge"] types: - completed diff --git a/proofs/nyt/ai-backlash/supports/idahonews.yaml b/proofs/ai-backlash/idahonews.yaml similarity index 100% rename from proofs/nyt/ai-backlash/supports/idahonews.yaml rename to proofs/ai-backlash/idahonews.yaml diff --git a/proofs/ht/china-zero-tarrif/supports/gov-cn.yaml b/proofs/china-zero-tarrif/gov-cn.yaml similarity index 100% rename from proofs/ht/china-zero-tarrif/supports/gov-cn.yaml rename to proofs/china-zero-tarrif/gov-cn.yaml diff --git a/proofs/al-jazeera/eu-sa-trade-deal/supports/policy-trade-europa.yaml b/proofs/eu-sa-trade-deal/policy-trade-europa.yaml similarity index 100% rename from proofs/al-jazeera/eu-sa-trade-deal/supports/policy-trade-europa.yaml rename to proofs/eu-sa-trade-deal/policy-trade-europa.yaml diff --git a/proofs/bbc/fifa-italy-iran/supports/theweek.yaml b/proofs/fifa-italy-iran/theweek.yaml similarity index 100% rename from proofs/bbc/fifa-italy-iran/supports/theweek.yaml rename to proofs/fifa-italy-iran/theweek.yaml diff --git a/proofs/bbc/fifa-wc-caution/supports/amnestyusa.yaml b/proofs/fifa-wc-caution/amnestyusa.yaml similarity index 100% rename from proofs/bbc/fifa-wc-caution/supports/amnestyusa.yaml rename to proofs/fifa-wc-caution/amnestyusa.yaml diff --git a/proofs/ht/india-gdp/supports/gov-in.yaml b/proofs/india-gdp/gov-in.yaml similarity index 100% rename from proofs/ht/india-gdp/supports/gov-in.yaml rename to proofs/india-gdp/gov-in.yaml diff --git a/proofs/scmp/moore-threads-share/supports/investing.yaml b/proofs/moore-threads-share/investing.yaml similarity index 100% rename from proofs/scmp/moore-threads-share/supports/investing.yaml rename to proofs/moore-threads-share/investing.yaml diff --git a/proofs/nyt/pentagon-google-ai/supports/war-gov.yaml b/proofs/pentagon-google-ai/war-gov.yaml similarity index 100% rename from proofs/nyt/pentagon-google-ai/supports/war-gov.yaml rename to proofs/pentagon-google-ai/war-gov.yaml diff --git a/proofs/al-jazeera/us-tarrifs/supports/economic-times.yaml b/proofs/us-tarrifs/economic-times.yaml similarity index 100% rename from proofs/al-jazeera/us-tarrifs/supports/economic-times.yaml rename to proofs/us-tarrifs/economic-times.yaml