Features/ml classification - Bioclip v2

.gitignore

@@ -63,7 +63,8 @@ cover/
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
-
+# autres
+test.py
 # Flask stuff:
 instance/
 .webassets-cache
@@ -170,4 +171,8 @@ images/
 labels/
 failed_downloads.csv
 metadata.csv
-data.yaml
+data.yaml
+
+# orchestrateur - tests Timothée
+orchestrateur/
+flows/

biolit/__init__.py

@@ -7,6 +7,18 @@
 RAWDIR = DATADIR / "raw"
 EXPORTDIR = DATADIR / "exports"
 
+DATA_GOUV_CONTOUR_COMMUNES_URL = (
+    "https://www.data.gouv.fr/api/1/datasets/r/00c0c560-3ad1-4a62-9a29-c34c98c3701e"
+)
+DATA_GOUV_INFO_COMMUNES_URL = (
+    "https://www.data.gouv.fr/api/1/datasets/r/f5df602b-3800-44d7-b2df-fa40a0350325"
+)
+WORLD_COAST_LINES_URL = (
+    "https://osmdata.openstreetmap.de/download/coastlines-split-4326.zip"
+)
+# /!\ URL emporaire de l'archive le temps que le museum corrige sa cybersec...
+TAXREFURL = "https://assets.patrinat.fr/files/referentiel/TAXREF_v18_2025.zip"
+
 
 mpl.rcParams["axes.spines.right"] = False
 mpl.rcParams["axes.spines.top"] = False

biolit/export_api.py

@@ -0,0 +1,89 @@
+import requests
+import polars as pl
+import structlog
+import re
+import os
+
+LOGGER = structlog.get_logger()
+
+# ------------------------------
+# FETCH API
+# ------------------------------
+def fetch_biolit_from_api():
+
+    url = os.getenv("BIOLIT_API_URL")
+
+    response = requests.get(url)
+    response.raise_for_status()
+
+    data = response.json()
+
+    print(f"{len(data)} observations récupérées")
+    return data
+
+# ------------------------------
+# RENAME OF COLUMNS
+# ------------------------------
+
+
+def normalize_column_name(col: str) -> str:
+    """Convertit les noms API en snake_case propre FR"""
+    col = col.lower()
+    col = col.replace("-", "_")
+    col = col.replace(" ", "_")
+    col = col.replace("é", "e").replace("è", "e").replace("à", "a")
+    col = col.replace("ù", "u").replace("ô", "o")
+    col = re.sub(r"[^a-z0-9_]", "", col)
+    return col
+
+
+COLUMN_MAPPING = {
+    "id": "id_observation",
+    "date": "date_observation",
+    "link": "lien_observation",
+    "author": "observateur",
+    "_url_sortie": "url_sortie",
+    "espece-identifiee": "espece_identifiee",
+    "heure-debut": "heure_debut",
+    "heure-fin": "heure_fin",
+    "latitude": "latitude",
+    "longitude": "longitude",
+    "photos": "photos",
+    "relais": "relais",
+    "espece_id": "id_espece",
+    "espece": "nom_scientifique",
+    "common": "nom_commun",
+    "categorie-programme": "categorie_programme",
+    "programme": "programme",
+}
+
+
+# ------------------------------
+# ADAPT API -> PARQUET
+# ------------------------------
+def adapt_api_to_dataframe(data: list) -> pl.DataFrame:
+    rows = []
+
+    for item in data:
+        new_row = {}
+
+        for key, value in item.items():
+            # mapping si connu, sinon normalisation auto
+            new_key = COLUMN_MAPPING.get(key, normalize_column_name(key))
+            new_row[new_key] = value
+
+        rows.append(new_row)
+
+    df = pl.DataFrame(rows)
+
+    return df
+
+
+# ------------------------------
+# LOAD (Fetch + Adapt)
+# ------------------------------
+def load_biolit_from_api() -> pl.DataFrame:
+    raw_data = fetch_biolit_from_api()
+    df = adapt_api_to_dataframe(raw_data)
+    return df
+