Add JSON-LD generation script with NRP AI support

.env.example

@@ -0,0 +1,28 @@
+# API Keys for JSON-LD Generation Script
+# Copy this file to .env and fill in your actual API keys
+# The .env file is gitignored and will not be committed
+
+# Google Gemini API Key (default service - uses URL Context Tool to browse URLs directly)
+# Get your key from: https://aistudio.google.com/apikey
+# Free tier available with .edu email
+# Default model: gemini-2.0-flash
+GEMINI_API_KEY=your-gemini-api-key-here
+
+# NRP (National Research Platform) API Key
+# Get your key from: https://nrp.ai/documentation/userdocs/ai/llm-managed/
+# Available NRP models: qwen3, llama3-sdsc, gpt-oss, gorilla, olmo, gemma3, kimi, etc.
+# Default model: qwen3
+# Note: NRP fetches HTML and extracts text before sending to AI
+NRP_API_KEY=your-nrp-api-key-here
+
+# OpenAI/ChatGPT API Key
+# Get your key from: https://platform.openai.com/api-keys
+# Default model: gpt-4o
+# Note: OpenAI fetches HTML and extracts text before sending to AI
+OPENAI_API_KEY=your-openai-api-key-here
+
+# Anthropic (Claude) API Key
+# Get your key from: https://console.anthropic.com/
+# Default model: claude-3-5-sonnet-20241022
+# Note: Anthropic fetches HTML and extracts text before sending to AI
+ANTHROPIC_API_KEY=your-anthropic-api-key-here

.github/workflows/sitemap_resources.yaml

@@ -0,0 +1,91 @@
+name: Generate XML sitemap for JSON-LD resources
+
+on:
+  push:
+    branches:
+      - main
+      - master
+
+permissions:
+  contents: write
+
+jobs:
+  sitemap_job:
+    runs-on: ubuntu-latest
+    name: Generate sitemap for JSON-LD files
+    steps:
+    - uses: actions/setup-python@v2
+      with:
+         python-version: 3.x
+    - run: pip install mkdocs
+    - run: pip install mkdocs-schema-reader
+    - name: Checkout the repo
+      uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+
+    # Generate single sitemap for all JSON-LD files in data and collection directories
+    - name: Generate sitemap for all JSON-LD resources
+      id: sitemap_all
+      uses: cicirello/generate-sitemap@v1
+      with:
+        base-url-path: https://raw.githubusercontent.com/earthcube/communityCollections/refs/heads/${{ github.ref_name }}/data/objects/summoned
+        #              https://raw.githubusercontent.com/earthcube/communityCollections/refs/heads/gh-pages
+        path-to-root: data/objects/summoned
+        include-html: false
+        include-pdf: false
+        additional-extensions: jsonld json xml
+        exclude-paths: .git .github docs scripts crawler prompts .vscode
+    - name: Output sitemap stats
+      run: |
+        echo "sitemap-path = ${{ steps.sitemap_all.outputs.sitemap-path }}"
+        echo "url-count = ${{ steps.sitemap_all.outputs.url-count }}"
+        echo "excluded-count = ${{ steps.sitemap_all.outputs.excluded-count }}"
+
+    - name: Generate sitemap for just AI Generated JSON-LD resources
+      id: sitemap_generated
+      uses: cicirello/generate-sitemap@v1
+      with:
+        base-url-path: https://raw.githubusercontent.com/earthcube/communityCollections/refs/heads/${{ github.ref_name }}/data/objects/summoned/generated
+        path-to-root: data/objects/summoned/generated
+        include-html: false
+        include-pdf: false
+        additional-extensions: jsonld json xml
+        exclude-paths:
+          .git .github docs scripts crawler prompts .vscode
+
+    - name: Commit and push sitemaps to branch
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        ls -la data/objects/summoned/sitemap.xml data/objects/summoned/generated/sitemap.xml 2>/dev/null || true
+        git add data/objects/summoned/sitemap.xml data/objects/summoned/generated/sitemap.xml
+        git status
+        if ! git diff --staged --quiet; then
+          git commit -m "chore: update JSON-LD sitemaps"
+          git fetch origin ${{ github.ref_name }}
+          git rebase origin/${{ github.ref_name }}
+          git push origin HEAD:${{ github.ref_name }}
+        fi
+
+### WE MIGHT WANT TO DO INDIVIDUAL SITEMAPS
+#    - name: Generate sitemap for all JSON-LD resources
+#      id: sitemap_glim
+#      uses: cicirello/generate-sitemap@v1
+#      with:
+#        base-url-path: https://raw.githubusercontent.com/earthcube/communityCollections/refs/heads/${{ github.ref_name }}/data/objects/summoned/glim
+#        path-to-root: data/objects/summoned/glim
+#        include-html: false
+#        include-pdf: false
+#        additional-extensions: jsonld json xml
+#        exclude-paths:
+#          .git .github docs scripts crawler prompts .vscode
+
+####### MKDOCS
+    - run: mkdocs build --config-file mkdocs.yaml
+    - name: push to gh pages
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        branch: gh-pages
+        folder: .
+        clean: false

.github/workflows/validate_with_dataset_schema.yaml

@@ -0,0 +1,148 @@
+name: Validate JSON-LD files with Schema.org Dataset schema
+
+on:
+  push:
+    branches-ignore: [ 'gh-pages' ]
+  pull_request:
+    branches-ignore: [ 'gh-pages' ]
+
+jobs:
+  validate-jsonld-generated:
+    runs-on: ubuntu-latest
+    name: Validate generated JSON-LD files
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Find and validate generated JSON-LD files
+      run: |
+        python scripts/validate_jsonld_batch.py data/objects/summoned/generated
+
+  validate-jsonld-summoned:
+    runs-on: ubuntu-latest
+    name: Validate summoned JSON-LD files
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Find and validate summoned JSON-LD files
+      run: |
+        python - << 'RELAXED_VALIDATE'
+        import json, sys
+        from pathlib import Path
+        dir_ = Path("data/objects/summoned")
+        if not dir_.exists():
+            print("Directory not found, skipping."); sys.exit(0)
+        files = [f for f in dir_.rglob("*.jsonld") if "generated" not in str(f)]
+        if not files:
+            print("No JSON-LD files found."); sys.exit(0)
+        errs = []
+        for f in sorted(files):
+            try:
+                with open(f) as fp: data = json.load(fp)
+            except Exception as e:
+                errs.append(f"{f}: {e}"); continue
+            for k in ["@context", "@type", "name"]:
+                if k not in data: errs.append(f"{f}: missing {k}")
+            if "spatialCoverage" in data and isinstance(data["spatialCoverage"], dict):
+                geo = data["spatialCoverage"].get("geo", {})
+                if isinstance(geo, dict) and "box" in geo and isinstance(geo["box"], str):
+                    parts = geo["box"].strip().split()
+                    if len(parts) == 4:
+                        try:
+                            a,b,c,d = float(parts[0]),float(parts[1]),float(parts[2]),float(parts[3])
+                            if (-90<=b<=90 and -90<=d<=90): west,south,east,north = a,b,c,d
+                            else: south,west,north,east = a,b,c,d
+                            if not (-90<=south<=90 and -90<=north<=90 and -180<=west<=180 and -180<=east<=180):
+                                errs.append(f"{f}: box out of range")
+                        except ValueError: errs.append(f"{f}: invalid box numbers")
+                    elif len(parts) == 2:
+                        try:
+                            ws, en = parts[0].split(","), parts[1].split(",")
+                            if len(ws)==2 and len(en)==2:
+                                west,south = float(ws[0]),float(ws[1])
+                                east,north = float(en[0]),float(en[1])
+                                if not (-90<=south<=90 and -90<=north<=90 and -180<=west<=180 and -180<=east<=180):
+                                    errs.append(f"{f}: box out of range")
+                        except ValueError: errs.append(f"{f}: invalid box format")
+                    else: errs.append(f"{f}: box expected 2 or 4 numbers")
+        if errs:
+            for e in errs: print(e)
+            sys.exit(1)
+        print(f"All {len(files)} summoned JSON-LD file(s) validated.")
+        RELAXED_VALIDATE
+
+  validate-jsonld-all:
+    runs-on: ubuntu-latest
+    name: Validate all JSON-LD files
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Find and validate all JSON-LD files
+      run: |
+        python - << 'RELAXED_VALIDATE_ALL'
+        import json, sys
+        from pathlib import Path
+        dir_ = Path("data")
+        if not dir_.exists():
+            print("Directory not found, skipping."); sys.exit(0)
+        files = list(dir_.rglob("*.jsonld"))
+        if not files:
+            print("No JSON-LD files found."); sys.exit(0)
+        errs = []
+        for f in sorted(files):
+            try:
+                with open(f) as fp: data = json.load(fp)
+            except Exception as e:
+                errs.append(f"{f}: {e}"); continue
+            for k in ["@context", "@type", "name"]:
+                if k not in data: errs.append(f"{f}: missing {k}")
+            if "spatialCoverage" in data and isinstance(data["spatialCoverage"], dict):
+                geo = data["spatialCoverage"].get("geo", {})
+                if isinstance(geo, dict) and "box" in geo and isinstance(geo["box"], str):
+                    parts = geo["box"].strip().split()
+                    if len(parts) == 4:
+                        try:
+                            a,b,c,d = float(parts[0]),float(parts[1]),float(parts[2]),float(parts[3])
+                            if (-90<=b<=90 and -90<=d<=90): west,south,east,north = a,b,c,d
+                            else: south,west,north,east = a,b,c,d
+                            if not (-90<=south<=90 and -90<=north<=90 and -180<=west<=180 and -180<=east<=180):
+                                errs.append(f"{f}: box out of range")
+                        except ValueError: errs.append(f"{f}: invalid box numbers")
+                    elif len(parts) == 2:
+                        try:
+                            ws, en = parts[0].split(","), parts[1].split(",")
+                            if len(ws)==2 and len(en)==2:
+                                west,south = float(ws[0]),float(ws[1])
+                                east,north = float(en[0]),float(en[1])
+                                if not (-90<=south<=90 and -90<=north<=90 and -180<=west<=180 and -180<=east<=180):
+                                    errs.append(f"{f}: box out of range")
+                        except ValueError: errs.append(f"{f}: invalid box format")
+                    else: errs.append(f"{f}: box expected 2 or 4 numbers")
+        if errs:
+            for e in errs: print(e)
+            sys.exit(1)
+        print(f"All {len(files)} JSON-LD file(s) validated.")
+        RELAXED_VALIDATE_ALL

.gitignore

@@ -1 +1,21 @@
 .idea
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# Environment variables
+.env
+.env.local
+
+# CSV data files (downloaded from Google Sheets)
+datasets.csv
+
+# Generated JSON-LD files
+#data/objects/summoned/generated/

README.md

@@ -5,3 +5,7 @@
 
 Documentation, files and code related to the exposure of resource on the web for indexing.
 
+Sitemaps (generated by [sitemap_resources.yaml](.github/workflows/sitemap_resources.yaml) on push to `master` / `main` / feature branches):
+
+- **All JSON-LD under data/objects/summoned:** [GitHub Pages](https://earthcube.github.io/communityCollections/data/objects/summoned/sitemap.xml) · [Raw (e.g. master)](https://raw.githubusercontent.com/earthcube/communityCollections/master/data/objects/summoned/sitemap.xml)
+- **AI-generated JSON-LD only:** [GitHub Pages](https://earthcube.github.io/communityCollections/data/objects/summoned/generated/sitemap.xml) · [Raw (e.g. master)](https://raw.githubusercontent.com/earthcube/communityCollections/master/data/objects/summoned/generated/sitemap.xml)