Fix/integration test runner

.github/workflows/run_tests.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches: [ "*" ]
   push:
-    branches: [ "main", "development", "release/*", "feature-stock-tracking" ]
+    branches: [ "main", "development", "release/*" ]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
@@ -16,35 +16,5 @@ jobs:
         working-directory: app/server
     steps:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-    - name: Build base images
-      run: docker compose build
-    - name: Build test images
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.test.yml build
-    - name: Start backend services
-      run: docker compose -f docker-compose.yml up -d database auth user analytics market api-gateway
     - name: Run tests
-      run: |
-        failed=()
-        for service in \
-          auth-test \
-          user-test \
-          analytics-test \
-          market-test \
-          market-mutation-test \
-          market-integration-test
-        do
-          echo "::group::Running $service"
-          if ! docker compose -f docker-compose.yml -f docker-compose.test.yml run --rm "$service"; then
-            failed+=("$service")
-          fi
-          echo "::endgroup::"
-        done
-
-        if [ ${#failed[@]} -gt 0 ]; then
-          echo "Failed test services: ${failed[*]}"
-          exit 1
-        fi
-    - name: Stop services
-      if: always()
-      run: docker compose -f docker-compose.yml -f docker-compose.test.yml down -v --remove-orphans
+      run: ./run_tests.sh

app/client/src/App.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useState } from "react";
 import {
   BrowserRouter as Router,
   Navigate,
@@ -13,11 +13,6 @@ import DashboardPage from "./pages/DashboardPage.tsx";
 import MarketsPage from "./pages/MarketsPage";
 import AppLayout from "./components/AppLayout.tsx";
 import ProjectionPage from "./pages/ProjectionPage.tsx";
-import { syncPinnedMarketsResetKey } from "./utils/marketStorage";
-//import { loadSession, saveSession, clearSession } from "./utils/storage.ts";
-//import { requestAuth } from "./api/AuthAPI";
-//import { resolveUserFromToken } from "./utils/token";
-// import type { AuthApiResponse, AuthSession, AuthUser } from "./pages/authTypes";
 
 const SESSION_STORAGE_KEY = "finus-session";
 const API_BASE_URL =
@@ -142,35 +137,6 @@ function App() {
     loadSession(),
   );
 
-  useEffect(() => {
-    let cancelled = false;
-
-    async function syncServerResetKey() {
-      try {
-        const response = await fetch(`${API_BASE_URL}/client-state/reset-key`);
-        const data = (await response.json().catch(() => null)) as {
-          resetKey?: unknown;
-        } | null;
-
-        if (cancelled || !response.ok) {
-          return;
-        }
-
-        if (typeof data?.resetKey === "string" && data.resetKey.length > 0) {
-          syncPinnedMarketsResetKey(data.resetKey);
-        }
-      } catch {
-        // Ignore reset-key sync failures and keep the client usable offline.
-      }
-    }
-
-    void syncServerResetKey();
-
-    return () => {
-      cancelled = true;
-    };
-  }, []);
-
   function handleAuthSuccess(token: string, fallbackUser: Partial<AuthUser>) {
     const nextSession: AuthSession = {
       token,

app/client/src/utils/marketStorage.ts

@@ -3,7 +3,6 @@ import { loadSession } from "@/utils/storage";
 
 const PINNED_MARKETS_STORAGE_KEY_PREFIX = "finus-pinned-markets";
 const LEGACY_PINNED_MARKETS_STORAGE_KEY = PINNED_MARKETS_STORAGE_KEY_PREFIX;
-const SERVER_RESET_KEY_STORAGE_KEY = "finus-server-reset-key";
 const PINNED_MARKETS_CLEARED_EVENT = "finus:pinned-markets-cleared";
 
 function resolvePinnedMarketsStorageKey(): string | null {
@@ -26,21 +25,6 @@ function clearLegacyPinnedMarkets() {
   localStorage.removeItem(LEGACY_PINNED_MARKETS_STORAGE_KEY);
 }
 
-function clearAllPinnedMarkets() {
-  clearLegacyPinnedMarkets();
-
-  for (let index = localStorage.length - 1; index >= 0; index -= 1) {
-    const key = localStorage.key(index);
-    if (!key) {
-      continue;
-    }
-
-    if (key.startsWith(`${PINNED_MARKETS_STORAGE_KEY_PREFIX}:`)) {
-      localStorage.removeItem(key);
-    }
-  }
-}
-
 function loadPinnedMarkets(): PinnedMarketInstrument[] {
   clearLegacyPinnedMarkets();
 
@@ -84,23 +68,9 @@ function savePinnedMarkets(items: PinnedMarketInstrument[]) {
   localStorage.setItem(storageKey, JSON.stringify(items));
 }
 
-function syncPinnedMarketsResetKey(resetKey: string): boolean {
-  const previousResetKey = localStorage.getItem(SERVER_RESET_KEY_STORAGE_KEY);
-  if (previousResetKey === resetKey) {
-    return false;
-  }
-
-  clearAllPinnedMarkets();
-  localStorage.setItem(SERVER_RESET_KEY_STORAGE_KEY, resetKey);
-  window.dispatchEvent(new Event(PINNED_MARKETS_CLEARED_EVENT));
-  return true;
-}
-
 export {
-  clearAllPinnedMarkets,
   loadPinnedMarkets,
   PINNED_MARKETS_CLEARED_EVENT,
   savePinnedMarkets,
   PINNED_MARKETS_STORAGE_KEY_PREFIX,
-  syncPinnedMarketsResetKey,
 };

app/server/.env

@@ -1,6 +1,8 @@
 #THIS IS NOT .gitignored'D! 
 #DON'T PUT REAL SECRETS IN HERE!!!
 
+API_GATEWAY_PORT=3000
+
 #CI
 DOCKER_HUB_USERNAME=measureonecodetwice
 RELEASE_VERSION=local

app/server/docker-compose.test.yml

@@ -1,9 +1,6 @@
-#This compose file runs all the tests. 
-#It depends on the main compose file being built
-#as all the test images inherit from the regular images.
-#you can use the following command to run the tests
-#docker compose build -q && docker compose -f docker-compose.yml -f docker-compose.test.yml build -q && docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-failure
+#Run with ./run-tests.sh
 services:
+    # ------ UNIT TESTS ------ 
     auth-test:
       build: 
         context: services/ts
@@ -19,56 +16,39 @@ services:
         context: services/python/analytics
         dockerfile: Dockerfile.test
 
-    # user-integration-test:#uncomment to run integration tests locally as it likely does not work properly in CI/CD
-    #   build:
-    #     context: ./services/ts
-    #     dockerfile: user/Dockerfile.integration.test
-    #   env_file:
-    #     - ./default_container.env
-    #   environment:
-    #     - JWT_SECRET
-
-    #   depends_on:
-    #     database:
-    #       condition: service_healthy
-    #     api-gateway: 
-    #       condition: service_started
+    market-test:
+      build:
+        context: services/python/market
+        dockerfile: Dockerfile.test
 
+    # # ------ MUTATION TESTS ------ 
     # user-mutation-test:
     #   build:
     #     context: ./services/ts
     #     dockerfile: user/Dockerfile.mutation.test
-    market-test:
-      build:
-        context: services/python/market
-        dockerfile: Dockerfile.test
 
     market-mutation-test:
       build:
         context: services/python/market
         dockerfile: Dockerfile.mutation.test
 
+    # MISSING
+    #  ANALYTICS
+    #  AUTH
+
+    # ------ INTEGRATION TESTS ------ 
     market-integration-test:
       build:
         context: services/python/market
         dockerfile: Dockerfile.integration.test
       env_file:
         - ./default_container.env
-      depends_on:
-        market:
-          condition: service_started
 
-    # user-integration-test:
-    #   build:
-    #     context: ./services/ts
-    #     dockerfile: user/Dockerfile.integration.test
-    #   env_file:
-    #     - ./default_container.env
-    #   environment:
-    #     - JWT_SECRET
-
-    #   depends_on:
-    #     database:
-    #       condition: service_healthy
-    #     api-gateway: 
-    #       condition: service_started
+    user-integration-test:
+      build:
+        context: ./services/ts
+        dockerfile: user/Dockerfile.integration.test
+      env_file:
+        - ./default_container.env
+      environment:
+        - JWT_SECRET

app/server/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       environment:
         - NODE_ENV=development
       ports:
-        - "3000:8000"
+        - "$API_GATEWAY_PORT:8000"
       volumes:
         - client_state_data:/var/lib/finus-client-state
       depends_on:

app/server/run_tests.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+quitting=false
+quit() {
+    if ! $quitting; then
+        quitting=true
+        cleanup
+        exit 0
+    fi
+}
+cleanup() {
+    echo "Cleaning up..."
+    docker compose -f docker-compose.test.yml -f docker-compose.yml down
+}
+trap quit SIGINT
+
+#--- Build images ---
+echo "Building app images..."
+docker compose build -q
+status=$?
+if [ $status -ne 0 ]; then
+    echo "Failed to build services"
+    exit $status
+fi
+
+echo "Building test images..."
+docker compose -f docker-compose.test.yml build -q
+status=$?
+if [ $status -ne 0 ]; then
+    echo "Failed to build test images"
+    exit $status
+fi
+
+#--- Launch services ---
+docker compose up -d
+
+#--- Wait for services to be healthy ---
+source .env
+
+is_app_healthy() { 
+    echo $(curl localhost:$API_GATEWAY_PORT/health 2> /dev/null | grep -Pc '^{(\"[^\"]*\":\"ok\",?)*}$')
+}
+
+curr_checks=0
+max_checks=10
+while [[  healthy=$(is_app_healthy) -ne 1 ]] && [ $curr_checks -lt $max_checks ]; do
+    curr_checks=$(($curr_checks + 1))
+    sleep 2.5
+    echo "App unhealthy. Retrying ($curr_checks of $max_checks)"
+done
+
+if [ $healthy -ne 1 ]; then
+    echo "App unhealthy after $max_checks retries. Exiting"
+    exit 1
+fi
+
+#--- Run tests ---
+docker compose -f docker-compose.test.yml up --abort-on-container-failure
+test_status=$?
+
+if [ $test_status -eq 0 ]; then
+    echo '+--------------------+'
+    echo '| ✅ All tests pass! |'
+    echo '+--------------------+'
+else 
+    echo '+--------------------+'
+    echo '|  ❌ Tests failed   |'
+    echo '+--------------------+'
+
+fi
+cleanup
+
+exit $test_status

app/server/services/python/analytics/src/dependencies.py

@@ -9,15 +9,16 @@
 SECRET_KEY = os.getenv('JWT_SECRET')
 ALGORITHM = 'HS256'
 
-def get_db_connection():
+def get_db_connection(timeout: int = 10):
     #check env vars
     if not all([os.getenv("MYSQL_HOST"), os.getenv("MYSQL_USER"), os.getenv("MYSQL_PASSWORD"), os.getenv("DB_NAME")]):
         raise HTTPException(status_code=500, detail="Missing environment variables")
     return mysql.connector.connect(
         host=os.getenv("MYSQL_HOST"),
         user=os.getenv("MYSQL_USER"),
         password=os.getenv("MYSQL_PASSWORD"),
-        database=os.getenv("DB_NAME")
+        database=os.getenv("DB_NAME"),
+        connection_timeout=timeout
     )
 
 async def get_current_user(authorization: Optional[str] = Header(None)):
@@ -35,4 +36,4 @@ async def get_current_user(authorization: Optional[str] = Header(None)):
         return int(user_id)
 
     except Exception as e:
-        raise HTTPException(status_code=401, detail=str(e))#general exception for simpler unit
+        raise HTTPException(status_code=401, detail=str(e))#general exception for simpler unit