Smart Grid Transformer Integration & Automated Scenario Generation Pipeline

.env.public

@@ -4,6 +4,7 @@ COUCHDB_USERNAME=admin
 COUCHDB_PASSWORD=password
 IOT_DBNAME=chiller                                                             
 WO_DBNAME=workorder
+TRANSFORMER_DBNAME=transformer
 
 # ── IBM WatsonX (plan-execute runner) ────────────────────────────────────────
 WATSONX_APIKEY=

.gitignore

@@ -201,6 +201,10 @@ mcp/couchdb/sample_data/bulk_docs.json
 .env
 mcp/servers/tsfm/artifacts/tsfm_models/
 src/tmp/
+src/scenarios/tmp/
+
+# ignore generated scenario outputs and manual scenario analyses
+generated/
 
 # Observability artifacts (OTLP-JSON traces + per-run trajectory JSON).
 traces/

pyproject.toml

@@ -12,6 +12,7 @@ description = "Add your description here"
 readme = "INSTRUCTIONS.md"
 requires-python = ">=3.12"
 dependencies = [
+    "certifi>=2024.8.30",
     "couchdb3>=2.0.2",
     "fastmcp>=2.14.5",
     "mcp[cli]>=1.26.0",
@@ -29,6 +30,9 @@ dependencies = [
     "langchain-openai>=1.1.0",
     "python-dotenv>=1.0",
     "scipy>=1.10.0",
+    "scikit-learn>=1.8.0",
+    "datasets>=4.8.4",
+    "pypdf>=6.9.2",
 ]
 
 [project.scripts]

src/couchdb/couchdb_setup.sh

@@ -49,5 +49,18 @@ else
   echo "⚠️ $VIBRATION_FILE not found, skipping vibration data."
 fi
 
+# Load transformer sample data (Transformer_01) into a dedicated database
+TRANSFORMER_FILE="/sample_data/iot/bulk_docs_transformer.json"
+if [ -f "$TRANSFORMER_FILE" ]; then
+  echo "Loading transformer data..."
+  COUCHDB_URL="http://localhost:5984" \
+    python3 /couchdb/init_asset_data.py \
+      --data-file "$TRANSFORMER_FILE" \
+      --db "${TRANSFORMER_DBNAME:-transformer}" \
+      --drop
+else
+  echo "⚠️ $TRANSFORMER_FILE not found, skipping transformer data."
+fi
+
 echo "✅ All databases initialised."
 tail -f /dev/null

src/couchdb/docker-compose.yaml

@@ -7,6 +7,7 @@ services:
       IOT_DBNAME: chiller
       WO_DBNAME: workorder
       VIBRATION_DBNAME: vibration
+      TRANSFORMER_DBNAME: transformer
     ports:
       - "5984:5984"
     volumes:

src/couchdb/sample_data/iot/bulk_docs_transformer.json

@@ -0,0 +1,172 @@
+[
+  {
+    "asset_id": "Transformer 1",
+    "timestamp": "2024-01-15T00:00:00.000000",
+    "hydrogen": 2845,
+    "oxygen": 5860,
+    "nitrogen": 27842,
+    "methane": 7406,
+    "co": 32,
+    "co2": 1344,
+    "ethylene": 16684,
+    "ethane": 5467,
+    "acetylene": 7,
+    "dbds": 19,
+    "power_factor": 1,
+    "interfacial_v": 45,
+    "dielectric_rigidity": 55,
+    "water_content": 0,
+    "vl1": 110,
+    "vl2": 115,
+    "vl3": 108,
+    "il1": 100,
+    "il2": 105,
+    "il3": 95,
+    "vl12": 225,
+    "vl23": 220,
+    "vl31": 218,
+    "inut": 5,
+    "rated_mva": 100,
+    "wti": 80,
+    "oti": 85,
+    "ati": 78,
+    "oti_a": 2,
+    "oti_t": 1
+  },
+  {
+    "asset_id": "Transformer 1",
+    "timestamp": "2024-01-16T00:00:00.000000",
+    "hydrogen": 2950,
+    "oxygen": 5700,
+    "nitrogen": 28000,
+    "methane": 7200,
+    "co": 30,
+    "co2": 1300,
+    "ethylene": 16500,
+    "ethane": 5400,
+    "acetylene": 6,
+    "dbds": 18,
+    "power_factor": 1.2,
+    "interfacial_v": 44,
+    "dielectric_rigidity": 53,
+    "water_content": 1,
+    "vl1": 112,
+    "vl2": 114,
+    "vl3": 109,
+    "il1": 102,
+    "il2": 103,
+    "il3": 96,
+    "vl12": 226,
+    "vl23": 221,
+    "vl31": 219,
+    "inut": 6,
+    "rated_mva": 100,
+    "wti": 82,
+    "oti": 87,
+    "ati": 79,
+    "oti_a": 2,
+    "oti_t": 1
+  },
+  {
+    "asset_id": "Transformer 1",
+    "timestamp": "2024-01-17T00:00:00.000000",
+    "hydrogen": 2800,
+    "oxygen": 5900,
+    "nitrogen": 27700,
+    "methane": 7300,
+    "co": 35,
+    "co2": 1350,
+    "ethylene": 16700,
+    "ethane": 5500,
+    "acetylene": 8,
+    "dbds": 20,
+    "power_factor": 0.9,
+    "interfacial_v": 46,
+    "dielectric_rigidity": 56,
+    "water_content": 0,
+    "vl1": 109,
+    "vl2": 116,
+    "vl3": 107,
+    "il1": 101,
+    "il2": 104,
+    "il3": 94,
+    "vl12": 224,
+    "vl23": 219,
+    "vl31": 217,
+    "inut": 5,
+    "rated_mva": 100,
+    "wti": 79,
+    "oti": 84,
+    "ati": 77,
+    "oti_a": 1,
+    "oti_t": 1
+  },
+  {
+    "asset_id": "Transformer 1",
+    "timestamp": "2024-01-18T00:00:00.000000",
+    "hydrogen": 2900,
+    "oxygen": 5800,
+    "nitrogen": 27950,
+    "methane": 7350,
+    "co": 33,
+    "co2": 1325,
+    "ethylene": 16600,
+    "ethane": 5450,
+    "acetylene": 7,
+    "dbds": 19,
+    "power_factor": 1.1,
+    "interfacial_v": 45,
+    "dielectric_rigidity": 54,
+    "water_content": 1,
+    "vl1": 111,
+    "vl2": 115,
+    "vl3": 108,
+    "il1": 100,
+    "il2": 106,
+    "il3": 95,
+    "vl12": 225,
+    "vl23": 220,
+    "vl31": 218,
+    "inut": 5,
+    "rated_mva": 100,
+    "wti": 81,
+    "oti": 86,
+    "ati": 78,
+    "oti_a": 2,
+    "oti_t": 1
+  },
+  {
+    "asset_id": "Transformer 1",
+    "timestamp": "2024-01-19T00:00:00.000000",
+    "hydrogen": 2850,
+    "oxygen": 5850,
+    "nitrogen": 27850,
+    "methane": 7400,
+    "co": 32,
+    "co2": 1340,
+    "ethylene": 16650,
+    "ethane": 5470,
+    "acetylene": 7,
+    "dbds": 19,
+    "power_factor": 1,
+    "interfacial_v": 45,
+    "dielectric_rigidity": 55,
+    "water_content": 0,
+    "vl1": 110,
+    "vl2": 115,
+    "vl3": 108,
+    "il1": 100,
+    "il2": 105,
+    "il3": 95,
+    "vl12": 225,
+    "vl23": 220,
+    "vl31": 218,
+    "inut": 5,
+    "rated_mva": 100,
+    "wti": 80,
+    "oti": 85,
+    "ati": 78,
+    "oti_a": 2,
+    "oti_t": 1
+  }
+]

src/llm/base.py

@@ -8,34 +8,44 @@
 
 @dataclass(frozen=True)
 class LLMResult:
-    """Return type for :meth:`LLMBackend.generate_with_usage`.
+    """Return type for :meth:`LLMBackend.generate` / :meth:`generate_with_usage`.
 
-    ``input_tokens`` / ``output_tokens`` are ``0`` when the backend can't
-    report usage (e.g. mocks in unit tests).
+    ``input_tokens`` / ``output_tokens`` / ``total_tokens`` are ``0`` when the
+    backend can't report usage (e.g. mocks in unit tests). ``total_tokens``
+    should reflect the provider total when available; otherwise callers may use
+    ``input_tokens + output_tokens``.
     """
-
     text: str
     input_tokens: int = 0
     output_tokens: int = 0
+    total_tokens: int = 0
 
 
 class LLMBackend(ABC):
     """Abstract interface for LLM backends."""
 
     @abstractmethod
-    def generate(self, prompt: str, temperature: float = 0.0) -> str:
-        """Generate text given a prompt."""
+    def generate(
+        self,
+        prompt: str,
+        temperature: float = 0.0,
+        max_tokens: int | None = None,
+    ) -> LLMResult:
+        """Generate text given a prompt; includes usage when the backend provides it."""
         ...
 
     def generate_with_usage(
-        self, prompt: str, temperature: float = 0.0
+        self,
+        prompt: str,
+        temperature: float = 0.0,
+        max_tokens: int | None = None,
     ) -> LLMResult:
         """Generate text and report token usage.
 
-        Default impl delegates to :meth:`generate` and reports zero usage —
-        backends that can surface counts (e.g. LiteLLM) should override.
+        Default impl delegates to :meth:`generate` (same fields as usage-aware
+        backends).
         """
-        return LLMResult(text=self.generate(prompt, temperature))
+        return self.generate(prompt, temperature, max_tokens)
 
     @property
     def model_id(self) -> str:

src/llm/litellm.py

@@ -14,10 +14,12 @@
 
 from __future__ import annotations
 
+import logging
 import os
 
 from .base import LLMBackend, LLMResult
 
+_log = logging.getLogger(__name__)
 _WATSONX_PREFIX = "watsonx/"
 
 
@@ -33,19 +35,30 @@ class LiteLLMBackend(LLMBackend):
     def __init__(self, model_id: str) -> None:
         self._model_id = model_id
 
-    def generate(self, prompt: str, temperature: float = 0.0) -> str:
-        return self.generate_with_usage(prompt, temperature).text
+    def generate(
+        self,
+        prompt: str,
+        temperature: float = 0.0,
+        max_tokens: int | None = None,
+    ) -> LLMResult:
+        return self.generate_with_usage(prompt, temperature, max_tokens)
 
     def generate_with_usage(
-        self, prompt: str, temperature: float = 0.0
+        self,
+        prompt: str,
+        temperature: float = 0.0,
+        max_tokens: int | None = None,
     ) -> LLMResult:
         import litellm
 
         kwargs: dict = {
             "model": self._model_id,
             "messages": [{"role": "user", "content": prompt}],
             "temperature": temperature,
-            "max_tokens": 2048,
+            "max_tokens": max_tokens or 2048,
+            # Explicit non-streaming: avoids union defaults and reduces Pydantic
+            # serializer warnings (Choices vs StreamingChoices) on model_dump.
+            "stream": False,
         }
 
         if self._model_id.startswith(_WATSONX_PREFIX):
@@ -59,8 +72,16 @@ def generate_with_usage(
 
         response = litellm.completion(**kwargs)
         usage = getattr(response, "usage", None)
+        prompt_n = int(getattr(usage, "prompt_tokens", 0) or 0)
+        completion_n = int(getattr(usage, "completion_tokens", 0) or 0)
+        raw_total = getattr(usage, "total_tokens", None) if usage else None
+        total_n = int(raw_total) if raw_total is not None else prompt_n + completion_n
+        text = response.choices[0].message.content
+        if text is None:
+            text = ""
         return LLMResult(
-            text=response.choices[0].message.content,
-            input_tokens=int(getattr(usage, "prompt_tokens", 0) or 0),
-            output_tokens=int(getattr(usage, "completion_tokens", 0) or 0),
+            text=text,
+            input_tokens=prompt_n,
+            output_tokens=completion_n,
+            total_tokens=total_n,
         )