Changement bdd

.github/workflows/deploy.yml

@@ -104,6 +104,9 @@ jobs:
             cd /var/www/umhma
             git fetch origin main
             git reset --hard origin/main
+            mkdir -p backend/src/config
+            curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
+              -o backend/src/config/global-bundle.pem
             cd backend
             npm ci --omit=dev
             pm2 restart umhma-api --update-env

backend/env.example

@@ -1,7 +1,14 @@
 PORT=4000
 JWT_SECRET=CHANGE_ME_OR_BOOT_FAILS_USE_AT_LEAST_32_CHARS
-DATABASE_URL=postgres://db_user:db_password@db_host:5432/db_name
-CORS_ALLOWED_ORIGINS=http://localhost:3000
+DB_HOST=your-rds-host.eu-west-3.rds.amazonaws.com
+DB_PORT=5432
+DB_NAME=UMHMA
+DB_USER=app_umhma
+DB_USER_ADMIN=umhma_admin
+AWS_REGION=eu-west-3
+CORS_ALLOWED_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
+# Optionnel : tests locaux avec URL Postgres classique
+# DATABASE_URL=postgres://db_user:db_password@db_host:5432/db_name
 # Si vous utilisez SSL (hébergement managé), décommentez la ligne ci-dessous
 # PGSSLMODE=require
 

backend/mistral_ocr.py

@@ -0,0 +1,241 @@
+# pip install mistralai python-dotenv
+
+
+
+from pathlib import Path
+from datetime import datetime
+from dotenv import load_dotenv
+import os
+import json
+import base64
+import mimetypes
+import re
+import html
+
+
+try:
+    from mistralai import Mistral
+except ImportError:
+    from mistralai.client import Mistral
+
+
+# ============================================================
+# CONFIG
+# ============================================================
+
+load_dotenv()
+
+BASE_DIR = Path("data/ocr")
+OUTPUT_DIR = BASE_DIR / "resultats_json"
+
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+SUPPORTED_EXTENSIONS = {
+    ".pdf",
+    ".png",
+    ".jpg",
+    ".jpeg"
+}
+
+
+# ============================================================
+# UTILS
+# ============================================================
+
+def clean_text(markdown: str) -> str:
+    if not markdown:
+        return ""
+
+    text = html.unescape(markdown)
+
+    text = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", text)
+    text = re.sub(r"</?[^>\n]+>", "", text)
+    text = re.sub(r"^#{1,6}\s*", "", text, flags=re.MULTILINE)
+    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
+
+    text = text.replace("**", "")
+    text = text.replace("__", "")
+    text = text.replace("*", "")
+
+    lines = []
+
+    for line in text.splitlines():
+        line = line.strip()
+        line = re.sub(r"\s+", " ", line)
+
+        if not line:
+            continue
+
+        lines.append(line)
+
+    return "\n".join(lines)
+
+
+def file_to_data_url(file_path: Path) -> str:
+    mime_type, _ = mimetypes.guess_type(file_path)
+
+    if mime_type is None:
+        if file_path.suffix.lower() == ".pdf":
+            mime_type = "application/pdf"
+        else:
+            mime_type = "application/octet-stream"
+
+    file_bytes = file_path.read_bytes()
+    encoded = base64.b64encode(file_bytes).decode("utf-8")
+
+    return f"data:{mime_type};base64,{encoded}"
+
+
+def response_to_dict(response):
+    if hasattr(response, "model_dump"):
+        return response.model_dump()
+
+    if hasattr(response, "dict"):
+        return response.dict()
+
+    if isinstance(response, dict):
+        return response
+
+    return json.loads(response.json())
+
+
+def save_json(result: dict) -> Path:
+    file_stem = Path(result["fichier"]).stem
+    output_path = OUTPUT_DIR / f"mistral_ocr_{file_stem}.json"
+
+    with output_path.open("w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    return output_path
+
+
+# ============================================================
+# MISTRAL OCR
+# ============================================================
+
+def mistral_ocr(file_path: str) -> dict:
+    path = Path(file_path)
+
+    if not path.exists():
+        raise FileNotFoundError(f"Fichier introuvable : {path}")
+
+    extension = path.suffix.lower()
+
+    if extension not in SUPPORTED_EXTENSIONS:
+        raise ValueError(
+            f"Extension non supportée : {extension}. "
+            f"Extensions acceptées : {sorted(SUPPORTED_EXTENSIONS)}"
+        )
+
+    api_key = os.getenv("MISTRAL_API_KEY")
+
+    if not api_key:
+        raise RuntimeError(
+            "Clé API Mistral introuvable. "
+            "Ajoute MISTRAL_API_KEY=ta_cle dans le fichier .env"
+        )
+
+    client = Mistral(api_key=api_key)
+
+    data_url = file_to_data_url(path)
+
+    if extension == ".pdf":
+        document = {
+            "type": "document_url",
+            "document_url": data_url
+        }
+    else:
+        document = {
+            "type": "image_url",
+            "image_url": data_url
+        }
+
+    ocr_response = client.ocr.process(
+        model="mistral-ocr-latest",
+        document=document,
+        include_image_base64=False
+    )
+
+    raw = response_to_dict(ocr_response)
+
+    pages = []
+
+    for i, page in enumerate(raw.get("pages", []), start=1):
+        markdown = page.get("markdown", "") or ""
+        texte = clean_text(markdown)
+
+        pages.append({
+            "page": i,
+            "texte": texte,
+            "texte_lignes": texte.splitlines(),
+            "markdown": markdown,
+            "images": page.get("images", []),
+            "tables": page.get("tables", []),
+            "dimensions": page.get("dimensions", {}),
+            "confidence_scores": page.get("confidence_scores")
+        })
+
+    return {
+        "fichier": path.name,
+        "chemin_source": str(path),
+        "extension": extension,
+        "type_extraction": "mistral_ocr",
+        "date_extraction": datetime.now().isoformat(timespec="seconds"),
+        "mode": "general",
+        "nb_pages": len(pages),
+        "pages": pages,
+        "model": raw.get("model", "mistral-ocr-latest"),
+        "usage_info": raw.get("usage_info", {})
+    }
+
+
+def process_folder(folder_path: str) -> list[dict]:
+    folder = Path(folder_path)
+
+    if not folder.exists():
+        raise FileNotFoundError(f"Dossier introuvable : {folder}")
+
+    results = []
+
+    for file_path in folder.iterdir():
+        if not file_path.is_file():
+            continue
+
+        if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+            continue
+
+        try:
+            print(f"Traitement : {file_path.name}")
+
+            result = mistral_ocr(str(file_path))
+            output = save_json(result)
+
+            print(f"OK : {output}")
+
+            results.append(result)
+
+        except Exception as e:
+            print(f"ERREUR {file_path.name} : {e}")
+
+    return results
+
+
+# ============================================================
+# TEST
+# ============================================================
+
+if __name__ == "__main__":
+
+    # # Modifier ici le chemin du PDF ou de l'image à tester
+    # fichier_test = "data/ocr/image_test/CarteID.pdf"
+
+    # result = mistral_ocr(fichier_test)
+
+    # output = save_json(result)
+
+    # print(json.dumps(result, ensure_ascii=False, indent=2))
+
+    # print(f"\nJSON sauvegardé : {output}")
+
+    # Pour faire l'OCR de tous les fichiers d'un dossier :
+    process_folder("data/ocr/image_test")