Feat: Log exception stack traces from executor tasks to API console (#825)

* Log exception stack traces from executor tasks to API console

* PR feedback

Author: vchan

web/server/console.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import traceback
 import typing as t
 import unittest
 
@@ -100,3 +101,14 @@ def log_test_results(
     def log_success(self, msg: str) -> None:
         self.queue.put_nowait(self._make_event(msg))
         self.stop_snapshot_progress()
+
+    def log_exception(self, exc: Exception) -> None:
+        """Log an exception."""
+        self.queue.put_nowait(
+            self._make_event(
+                {"details": str(exc), "traceback": traceback.format_exc()}, event="errors", ok=False
+            )
+        )
+
+
+api_console = ApiConsole()

web/server/main.py

@@ -6,11 +6,10 @@
 from fastapi.staticfiles import StaticFiles
 
 from web.server.api.endpoints import api_router
-from web.server.console import ApiConsole
+from web.server.console import api_console
 from web.server.watcher import watch_project
 
 app = FastAPI()
-api_console = ApiConsole()
 
 app.include_router(api_router, prefix="/api")
 WEB_DIRECTORY = pathlib.Path(__file__).parent.parent

web/server/utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import functools
 import io
 import traceback
 import typing as t
@@ -13,6 +14,7 @@
 from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
 
 from sqlmesh.core.context import Context
+from web.server.console import api_console
 from web.server.settings import get_context
 
 R = t.TypeVar("R")
@@ -26,8 +28,17 @@ def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
 
 async def run_in_executor(func: t.Callable[..., R], *args: t.Any) -> R:
     """Run in the default loop's executor"""
+
+    @functools.wraps(func)
+    def func_wrapper() -> R:
+        try:
+            return func(*args)
+        except Exception as e:
+            api_console.log_exception(e)
+            raise e
+
     loop = asyncio.get_running_loop()
-    return await loop.run_in_executor(None, func, *args)
+    return await loop.run_in_executor(None, func_wrapper)
 
 
 def validate_path(