feat(scheduled_jobs): add scheduler database support and improve SQLite configuration, get around the write lock when scheduler and Runner wants to write to the same DB

Introduced a new `scheduler_url` field in `DbSettings` for APScheduler job storage, allowing separate database management. Updated the `engine.py` to utilize the new scheduler database and removed unused SQLite connection settings. Enhanced the `scheduler.py` to create a dedicated engine for the scheduler database and ensure proper SQLite configuration for foreign key support. Updated logging to use a private logger for better encapsulation.

Author: danyalxahid-askui

.gitignore

@@ -166,8 +166,7 @@ reports/
 .DS_Store
 /chat
 /askui_chat.db
-/askui_chat.db-shm
-/askui_chat.db-wal
+/askui_scheduler.db
 .cache/
 
 bom.json

src/askui/chat/api/db/engine.py

@@ -2,7 +2,7 @@
 from sqlite3 import Connection as SQLite3Connection
 from typing import Any
 
-from sqlalchemy import Engine, create_engine, event
+from sqlalchemy import create_engine, event
 
 from askui.chat.api.dependencies import get_settings
 
@@ -14,12 +14,8 @@
 engine = create_engine(settings.db.url, connect_args=connect_args, echo=echo)
 
 
-@event.listens_for(Engine, "connect")
+@event.listens_for(engine, "connect")
 def set_sqlite_pragma(dbapi_conn: SQLite3Connection, connection_record: Any) -> None:  # noqa: ARG001
     cursor = dbapi_conn.cursor()
     cursor.execute("PRAGMA foreign_keys=ON")
-    # WAL mode - allows concurrent readers + writer
-    cursor.execute("PRAGMA journal_mode=WAL")
-    # Busy timeout - prevents SQLite from hanging on locked databases
-    cursor.execute("PRAGMA busy_timeout=10000")  # 10 seconds
     cursor.close()

src/askui/chat/api/scheduled_jobs/scheduler.py

@@ -6,17 +6,32 @@
 """
 
 import logging
+from sqlite3 import Connection as SQLite3Connection
 from typing import Any
 
 from apscheduler import AsyncScheduler
 from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore
+from sqlalchemy import create_engine, event
 
-from askui.chat.api.db.engine import engine
+from askui.chat.api.dependencies import get_settings
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
-# Module-level singleton data store (similar to engine pattern)
-_data_store: Any = SQLAlchemyDataStore(engine_or_url=engine)
+# Module-level settings for scheduler database
+_settings = get_settings()
+_connect_args = {"check_same_thread": False}
+_echo = _logger.isEnabledFor(logging.DEBUG)
+
+# Separate engine for scheduler database
+scheduler_engine = create_engine(
+    _settings.db.scheduler_url,
+    connect_args=_connect_args,
+    echo=_echo,
+)
+
+
+# Module-level singleton data store using separate scheduler database
+_data_store: Any = SQLAlchemyDataStore(engine_or_url=scheduler_engine)
 
 # Module-level singleton scheduler instance
 # - max_concurrent_jobs=1: only one job runs at a time (sequential execution)
@@ -27,6 +42,13 @@
 )
 
 
+@event.listens_for(scheduler_engine, "connect")
+def set_sqlite_pragma(dbapi_conn: SQLite3Connection, connection_record: Any) -> None:  # noqa: ARG001
+    cursor = dbapi_conn.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
 async def start_scheduler() -> None:
     """
     Start the scheduler to begin processing jobs.
@@ -38,10 +60,10 @@ async def start_scheduler() -> None:
     await scheduler.__aenter__()
     # Then start background processing of jobs
     await scheduler.start_in_background()
-    logger.info("Scheduler started in background")
+    _logger.info("Scheduler started in background")
 
 
 async def shutdown_scheduler() -> None:
     """Shut down the scheduler gracefully."""
     await scheduler.__aexit__(None, None, None)
-    logger.info("Scheduler shut down")
+    _logger.info("Scheduler shut down")

src/askui/chat/api/settings.py

@@ -18,12 +18,16 @@ class DbSettings(BaseModel):
         default_factory=lambda: f"sqlite:///{(Path.cwd().absolute() / 'askui_chat.db').as_posix()}",
         description="Database URL for SQLAlchemy connection",
     )
+    scheduler_url: str = Field(
+        default_factory=lambda: f"sqlite:///{(Path.cwd().absolute() / 'askui_scheduler.db').as_posix()}",
+        description="Database URL for APScheduler job storage",
+    )
     auto_migrate: bool = Field(
         default=True,
         description="Whether to run migrations automatically on startup",
     )
 
-    @field_validator("url")
+    @field_validator("url", "scheduler_url")
     @classmethod
     def validate_sqlite_url(cls, v: str) -> str:
         """Ensure only synchronous SQLite URLs are allowed."""