docker-compose.yml

networks:
  kafka-network:
    driver: bridge
  backend:
    driver: bridge
  gdelt-network:
    driver: bridge
  gdelt-shared-network:
    driver: bridge
  mcp-network:
    driver: bridge
  observability-network:
    driver: bridge

volumes:
  grafana-data:

x-common-service: &common-service
  build: firecrawl-webscraper/apps/api
  ulimits:
    nofile:
      soft: 65535
      hard: 65535
  networks:
    - backend
  extra_hosts:
    - "host.docker.internal:host-gateway"

x-common-env: &common-env
  REDIS_URL: ${REDIS_URL:-redis://redis:6379}
  REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
  PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3001/scrape}
  NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgres
  USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
  OPENAI_API_KEY: ${OPENAI_API_KEY}
  OPENAI_BASE_URL: ${OPENAI_BASE_URL}
  MODEL_NAME: ${MODEL_NAME}
  MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME}
  OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}
  SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
  BULL_AUTH_KEY: ${BULL_AUTH_KEY}
  TEST_API_KEY: ${TEST_API_KEY}
  POSTHOG_API_KEY: ${POSTHOG_API_KEY}
  POSTHOG_HOST: ${POSTHOG_HOST}
  SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
  SUPABASE_URL: ${SUPABASE_URL}
  SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
  SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
  SERPER_API_KEY: ${SERPER_API_KEY}
  SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}
  LOGGING_LEVEL: ${LOGGING_LEVEL}
  PROXY_SERVER: ${PROXY_SERVER}
  PROXY_USERNAME: ${PROXY_USERNAME}
  PROXY_PASSWORD: ${PROXY_PASSWORD}
  SEARXNG_ENDPOINT: ${SEARXNG_ENDPOINT}
  SEARXNG_ENGINES: ${SEARXNG_ENGINES}
  SEARXNG_CATEGORIES: ${SEARXNG_CATEGORIES}

services:
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    container_name: kafka
    networks:
      - kafka-network
      - observability-network
    volumes:
      - ./docker/kafka-entrypoint.sh:/docker/kafka-entrypoint.sh:ro
      - ./jmx/jmx_prometheus_javaagent.jar:/opt/jmx_exporter/jmx.jar
      - ./jmx/kafka.yml:/opt/jmx_exporter/kafka.yml
    #entrypoint: ["/usr/bin/kafka-topics","--bootstrap-server", "kafka:9092", "--delete", "--topic", "*"] # delete old topics data
    entrypoint: ["/bin/bash", "/docker/kafka-entrypoint.sh"]
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
      KAFKA_LISTENERS: "PLAINTEXT://kafka:9092,CONTROLLER://kafka:9093,PLAINTEXT_HOST://0.0.0.0:29092"
      # KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://10.77.149.189:9092,PLAINTEXT_HOST://localhost:29092'
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092" #Uncomment for using kafka host in docker network
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs"
      KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      # KAFKA_JMX_PORT: 9999
      # KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx.jar=9711:/opt/jmx_exporter/kafka.yml"
      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
    ports:
      - "9092:9092"
      - "29092:29092"
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "kafka-broker-api-versions --bootstrap-server kafka:9092 || exit 1",
        ]
      interval: 10s
      timeout: 10s
      retries: 10
      start_period: 30s

  # kafka-exporter:
  #   image: danielqsj/kafka-exporter
  #   container_name: kafka-exporter
  #   ports:
  #     - "9308:9308"
  #   environment:
  #     KAFKA_URIS: kafka:9092
  #     LOG_LEVEL: info
  #   depends_on:
  #     - kafka
  #   networks:
  #     - observability-network

  prometheus:
    image: prom/prometheus
    depends_on:
      - kafka
      # - kafka-exporter
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    networks:
      - observability-network

  grafana:
    image: grafana/grafana
    depends_on:
      - prometheus
    ports:
      - "3010:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_SECURITY_ADMIN_USER=admin
      - GF_USERS_ALLOW_SIGN_UP=false
      # - GF_SERVER_HTTP_PORT=3000
      - GF_SERVER_ROOT_URL=/grafana/
      - GF_SERVER_SERVE_FROM_SUB_PATH=true
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - grafana-data:/var/lib/grafana
    networks:
      - observability-network

  kafbat-ui:
    container_name: kafbat-ui
    image: ghcr.io/kafbat/kafka-ui:latest
    ports:
      - 8080:8080
    depends_on:
      - kafka
    networks:
      - kafka-network
      - observability-network
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
      FORWARD_HEADERS_STRATEGY: FRAMEWORK
      DYNAMIC_CONFIG_ENABLED: "true"

  # finnhub-stream-service:
  #   build: ./stock-data-pipelines/finnhub-stream-service/
  #   networks:
  #     - kafka-network
  #   depends_on:
  #     kafka:
  #       condition: service_healthy
  #   environment:
  #     KAFKA_BROKER: kafka:9092
  #     FINNHUB_API_KEY: ${FINNHUB_API_KEY}

  # kite-stream-service:
  #   build: ./stock-data-pipelines/kite-stream-service/
  #   networks:
  #     - kafka-network
  #   depends_on:
  #     kafka:
  #       condition: service_healthy
  #   environment:
  #     KAFKA_BROKER: kafka:9092
  #     KITE_API_KEY: ${KITE_API_KEY}
  #     KITE_ACCESS_TOKEN: ${KITE_ACCESS_TOKEN}
  #
  mcp-server:
    build:
      context: ./mcp-server
    networks:
      - kafka-network
      - mcp-network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      kafka:
        condition: service_healthy
      news-gold-pipeline:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      VECTOR_STORE_HOST: gdelt-gold-pipeline
      VECTOR_STORE_PORT: 8765
      SOCIAL_VECTOR_STORE_HOST: pathway-vector-ingestion-pipeline
      SOCIAL_VECTOR_STORE_PORT: 8766
    ports:
      - "8123:8123"
  #
  agentic-pipeline-chatbot:
    build:
      context: ./agentic-pipeline
      dockerfile: Dockerfile.chat
    networks:
      - backend
      - observability-network
      # - mcp-network
    environment:
      PATHWAY_MCP_URL: ${PATHWAY_MCP_URL}
      BACKEND_URL: ${BACKEND_URL}
      CHATBOT_SECRET_KEY: ${CHATBOT_SECRET_KEY}
      OPENAI_API_KEY: ${OPENAI_API_KEY}
    ports:
      - "8010:8010"
  #
  # agentic-pipeline-research:
  #   build:
  #     context: ./agentic-pipeline
  #     dockerfile: Dockerfile.research
  #   networks:
  #     - backend
  #     - observability-network
  #     - mcp-network
  #   environment:
  #     PATHWAY_MCP_URL: ${PATHWAY_MCP_URL}
  #     BACKEND_URL: ${BACKEND_URL}
  #     CHATBOT_SECRET_KEY: ${CHATBOT_SECRET_KEY}
  #     OPENAI_API_KEY: ${OPENAI_API_KEY}

  # kite-stream-service:
  #   build: ./stock-data-pipelines/kite-stream-service/
  #   networks:
  #     - kafka-network
  #   depends_on:
  #     kafka:
  #       condition: service_healthy
  #   environment:
  #     KAFKA_BROKER: kafka:9092
  #     KITE_API_KEY: ${KITE_API_KEY}
  #     KITE_ACCESS_TOKEN: ${KITE_ACCESS_TOKEN}

  replay-stream-service:
    build: ./stock-data-pipelines/replay-stream-service/
    networks:
      - kafka-network
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092

  stock-bronze-tier-pipeline:
    build: ./stock-data-pipelines/bronze-tier-pipeline/
    networks:
      - kafka-network
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092

  stock-silver-tier-pipeline:
    build: ./stock-data-pipelines/silver-tier-pipeline/
    networks:
      - kafka-network
    depends_on:
      kafka:
        condition: service_healthy
      # redis:
      #   condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      REDIS_HOST: redis
      REDIS_HOST_PORT: 6379
      MONGODB_URI: ${HIST_MONGODB_URI}

  stock-gold-tier-pipeline:
    build: ./stock-data-pipelines/gold-tier-pipeline/
    networks:
      - kafka-network
    depends_on:
      kafka:
        condition: service_healthy
      # redis:
      #   condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      REDIS_HOST: redis
      REDIS_HOST_PORT: 6379
      MONGODB_URI: ${HIST_MONGODB_URI}

  # stock-gold-tier-pipeline:
  #  build: ./stock-data-pipelines/gold-tier-pipeline/
  #  networks:
  #   - kafka-network
  #  depends_on:
  #   kafka:
  #    condition: service_healthy
  #  environment:
  #   KAFKA_BROKER: kafka:9092
  #   NVIDIA_VISIBLE_DEVICES: all
  #   NVIDIA_DRIVER_CAPABILITIES: compute,utility
  #  deploy:
  #    resources:
  #      reservations:
  #        devices:
  #          - driver: nvidia
  #            count: 1
  #            capabilities: [gpu]

  stock-api:
    build: ./stock-data-pipelines/backtesting-python
    networks:
      - kafka-network
      - backend
    ports:
      - "5001:5001"
    environment:
      - FLASK_ENV=development
      - FLASK_DEBUG=1
    restart: unless-stopped

  # NEWS DATA PIPELINE (GDELT) SERVICES

  gdelt-fetcher:
    build:
      context: ./news-data-pipeline
      dockerfile: Dockerfile.base
    container_name: gdelt-fetcher
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8014:8014" # Prometheus metrics
    environment:
      KAFKA_BROKER: kafka:9092
      RAW_GDELT_TOPIC: raw_gdelt_events
      GDELT_POLL_INTERVAL: 900 # 15 minutes
      FETCHER_PROMETHEUS_METRICS_PORT: 8014
    command: python gdelt_fetcher.py
    networks:
      - kafka-network
      - observability-network
    restart: unless-stopped

  news-bronze-pipeline:
    build:
      context: ./news-data-pipeline
      dockerfile: Dockerfile.base
    container_name: gdelt-bronze-pipeline
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8011:8011" # Prometheus metrics
    environment:
      KAFKA_BROKER: kafka:9092
      RAW_GDELT_TOPIC: raw_gdelt_events
      PROCESSED_GDELT_TOPIC: processed_gdelt_events
      KAFKA_GROUP_ID: gdelt-bronze-group
      OUTPUT_FILE: /data/processed_events.jsonl
      BRONZE_PROMETHEUS_METRICS_PORT: 8011
    command: python bronze_pipeline.py
    volumes:
      - ./data:/data
    networks:
      - kafka-network
      - observability-network
    restart: unless-stopped

  news-silver-pipeline:
    build:
      context: ./news-data-pipeline
      dockerfile: Dockerfile.base
    container_name: gdelt-silver-pipeline
    depends_on:
      kafka:
        condition: service_healthy
      news-bronze-pipeline:
        condition: service_started
    ports:
      - "8012:8012" # Prometheus metrics
    environment:
      KAFKA_BROKER: kafka:9092
      PROCESSED_GDELT_TOPIC: processed_gdelt_events
      ENRICHED_GDELT_TOPIC: enriched_gdelt_events
      KAFKA_GROUP_ID: gdelt-silver-group
      ENRICHED_FILE: /data/enriched_events.jsonl
      AGGREGATES_FILE: /data/country_aggregates.jsonl
      FIRECRAWL_URL: ${FIRECRAWL_URL}
      SCRAPE_TIMEOUT: 60
      SCRAPE_CACHE_DIR: /data/cache
      SKIP_SCRAPING: "false"
      SCRAPE_WORKERS: "10"
      BATCH_FLUSH_INTERVAL_MS: "5000"
      SCRAPE_BATCH_SIZE: "10"
      FROM_BRONZE_FILE: /data/FROM_BRONZE.jsonl
      TO_GOLD_FILE: /data/TO_GOLD.jsonl
      SILVER_PROMETHEUS_METRICS_PORT: 8012
    command: python silver_pipeline.py
    volumes:
      - ./data:/data
    networks:
      - kafka-network
      - backend
      - gdelt-shared-network
      - observability-network
    restart: unless-stopped

  #Pathway Vector Store for News Pipeline (replaces Qdrant)
  news-gold-pipeline:
    build:
      context: ./news-data-pipeline
      dockerfile: Dockerfile.gold
    container_name: gdelt-gold-pipeline
    ports:
      - "8765:8765"
      - "8013:8013" # Prometheus metrics
    depends_on:
      kafka:
        condition: service_healthy
      news-silver-pipeline:
        condition: service_started
    environment:
      KAFKA_BROKER: kafka:9092
      ENRICHED_GDELT_TOPIC: enriched_gdelt_events
      GOLD_OUTPUT_TOPIC: NEWS-GOLD-TIER
      KAFKA_GROUP_ID: gdelt-gold-group
      VECTOR_STORE_HOST: 0.0.0.0
      VECTOR_STORE_PORT: 8765
      EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2
      # IMPORTANT: Use separate directory for vector store persistence
      # Do NOT use /data/cache as that contains scrape cache files
      VECTOR_STORE_CACHE_DIR: /data/vector_cache
      BATCH_FLUSH_INTERVAL_MS: "5000"
      GOLD_PROMETHEUS_METRICS_PORT: 8013
      # Historical data settings
      HISTORICAL_DATA_ENABLED: ${HISTORICAL_DATA_ENABLED:-true}
      HISTORICAL_DATA_GDRIVE_URL: ${HISTORICAL_DATA_GDRIVE_URL}
      HISTORICAL_DATA_DOWNLOAD_DIR: /data/historical_data
      HISTORICAL_DATA_FORCE_REDOWNLOAD: ${HISTORICAL_DATA_FORCE_REDOWNLOAD:-false}
      HISTORICAL_DATA_MONTHS: ${HISTORICAL_DATA_MONTHS}
      HISTORICAL_DATA_MAX_FILES: ${HISTORICAL_DATA_MAX_FILES:-0}
      HISTORICAL_DATA_DAYS: ${HISTORICAL_DATA_DAYS:-0} # Number of recent days to load (0=all data, 5=last 5 days)
      HISTORICAL_DATA_COMPANY_ONLY: ${HISTORICAL_DATA_COMPANY_ONLY:-false} # Filter to company mentions only
      # Environment mode (dev/prod)
      ENVIRONMENT: ${ENVIRONMENT:-dev}
      # Local persistence for dev mode
      LOCAL_SNAPSHOT_DIR: /data/snapshots
      LOCAL_VECTOR_DIR: /data/vectors
    command: python gold_pipeline.py
    volumes:
      - ./data:/data
    networks:
      - kafka-network
      - mcp-network
      - gdelt-shared-network
      - observability-network
    restart: unless-stopped
    healthcheck:
      test:
        ["CMD-SHELL", "curl -f http://localhost:8765/v1/statistics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # AI Processor: Processes raw filings using Docling + GPT-4 for extraction
  pathway-filings-processor:
    build: ./filings-pipeline
    container_name: pathway-filings-processor
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      - KAFKA_BROKER=kafka:9092
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - PATHWAY_LICENSE_KEY=${PATHWAY_LICENSE_KEY}
    volumes:
      - ./filings-pipeline/pathway_cache:/app/pathway_cache
      - ./filings-pipeline/data:/app/data
    command: python pathway_processor.py --config config/config.yaml
    networks:
      - kafka-network
      - observability-network
    restart: unless-stopped

  # RAG API: Semantic search over processed filings
  pathway-filings-rag:
    build: ./filings-pipeline
    container_name: pathway-filings-rag
    depends_on:
      - pathway-filings-processor
    ports:
      - "8767:8767"
    environment:
      - KAFKA_BROKER=kafka:9092
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - PATHWAY_LICENSE_KEY=${PATHWAY_LICENSE_KEY}
    volumes:
      - ./filings-pipeline/pathway_cache:/app/pathway_cache
      - ./filings-pipeline/pathway_state:/app/pathway_state
      - ./filings-pipeline/data:/app/data
    command: python pathway_rag.py --config config/config.yaml --port 8767
    networks:
      - kafka-network
      - mcp-network
      - observability-network
    restart: unless-stopped
    healthcheck:
      test:
        ["CMD-SHELL", "curl -f http://localhost:8767/v1/statistics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # NSE Fetcher: Fetches filings from NSE every hour
  nse-filings-fetcher:
    build: ./filings-pipeline
    container_name: nse-filings-fetcher
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      - KAFKA_BROKER=kafka:9092
    volumes:
      - ./filings-pipeline/data:/app/data
    command: >
      bash -c "while true; do
        echo '=== Starting Hourly NSE Filings Fetch (Today Only) ===';
        python nse_fetcher.py --from-today;
        echo '=== Fetch Complete. Next run in 1 hour... ===';
        sleep 3600;
      done"
    networks:
      - kafka-network
      - observability-network
    restart: unless-stopped

  # COMMENTED OUT OLD DEFINITIONS

  # playwright-service:
  #   build: firecrawl-webscraper/apps/playwright-service-ts
  #   environment:
  #     PORT: 3001
  #     PROXY_SERVER: ${PROXY_SERVER}
  #     PROXY_USERNAME: ${PROXY_USERNAME}
  #     PROXY_PASSWORD: ${PROXY_PASSWORD}
  #     BLOCK_MEDIA: ${BLOCK_MEDIA}
  #   networks:
  #     - backend

  # firecrawl-api:
  #   <<: *common-service
  #   container_name: firecrawl-api
  #   networks:
  #     - backend
  #     - kafka-network
  #     - gdelt-shared-network
  #     - observability-network
  #   environment:
  #     <<: *common-env
  #     HOST: "0.0.0.0"
  #     PORT: ${INTERNAL_PORT:-3002}
  #     EXTRACT_WORKER_PORT: ${EXTRACT_WORKER_PORT:-3004}
  #     WORKER_PORT: ${WORKER_PORT:-3005}
  #     NUQ_WORKER_COUNT: '10'
  #     ENV: local
  #     REDIS_URL: redis://redis:6379
  #   depends_on:
  #     - redis
  #     - playwright-service
  #   ports:
  #     - "${PORT:-3002}:${INTERNAL_PORT:-3002}"
  #   command: node dist/src/harness.js --start-docker
  #
  # redis:
  #   image: redis:alpine
  #   networks:
  #     - backend
  #     - kafka-network
  #   command: redis-server --bind 0.0.0.0
  #   ports:
  #     - "6379:6379"
  #   healthcheck:
  #     test: ["CMD", "redis-cli", "ping"]
  #     interval: 10s
  #     timeout: 5s
  #     retries: 5

  # nuq-postgres:
  #   build: firecrawl-webscraper/apps/nuq-postgres
  #   environment:
  #     POSTGRES_USER: postgres
  #     POSTGRES_PASSWORD: postgres
  #     POSTGRES_DB: postgres
  #   networks:
  #     - backend
  #     - kafka-network
  #   ports:
  #     - "5433:5432"

  bronze-layer:
    build:
      context: ./social-media-web-scrapping
      dockerfile: Dockerfile.base
    container_name: bronze-layer-processor
    networks:
      - kafka-network
      - observability-network
    ports:
      - "8001:8000" # Prometheus metrics
    environment:
      - KAFKA_BROKER=kafka:9092
      - KAFKA_GROUP_ID=bronze-layer-processor
      - PRAW_API=stfQv-7DgTT_h6kveEkfFw
      - BRONZE_POST_TOPIC=bronze_reddit_posts
      - BRONZE_COMMENT_TOPIC=bronze_reddit_comments
      - BRONZE_METADATA_TOPIC=bronze_reddit_metadata
      - PROMETHEUS_METRICS_PORT=8000
    volumes:
      - ./:/app:cached,z
    depends_on:
      kafka:
        condition: service_healthy
    restart: unless-stopped
    command: python social-media-web-scrapping/bronze_layer.py
  
  silver-layer:
    build:
      context: ./social-media-web-scrapping
      dockerfile: Dockerfile.base
    container_name: silver-layer-processor
    networks:
      - kafka-network
      - observability-network
    ports:
      - "8003:8001" # Prometheus metrics
    environment:
      - KAFKA_BROKER=kafka:9092
      - KAFKA_GROUP_ID=silver-layer-processor
      - BRONZE_POST_TOPIC=bronze_reddit_posts
      - BRONZE_COMMENT_TOPIC=bronze_reddit_comments
      - SILVER_POST_TOPIC=silver_reddit_posts
      - SILVER_COMMENT_TOPIC=silver_reddit_comments
      - SILVER_PROMETHEUS_METRICS_PORT=8001
    volumes:
      - ./:/app:cached,z
    depends_on:
      kafka:
        condition: service_healthy
    restart: unless-stopped
    command: python social-media-web-scrapping/silver_layer.py
  
  gold-layer:
    build:
      context: ./social-media-web-scrapping
      dockerfile: Dockerfile.gold
    container_name: gold-layer-aggregator
    networks:
      - kafka-network
      - observability-network
    ports:
      - "8002:8002" # Prometheus metrics
    environment:
      - KAFKA_BROKER=kafka:9092
      - KAFKA_GROUP_ID=gold-layer-aggregator
      - BRONZE_METADATA_TOPIC=bronze_metadata
      - SILVER_POST_TOPIC=silver_reddit_posts
      - SILVER_COMMENT_TOPIC=silver_reddit_comments
      - GOLD_COMPANY_ANALYTICS_TOPIC=gold_company_analytics
      - GOLD_SECTOR_TOPIC=gold_sector_analytics
      - OUT_FILE_PATH=/app/gold_company_analytics.jsonl
      - GOLD_PROMETHEUS_METRICS_PORT=8002
    volumes:
      - ./:/app:cached,z
    depends_on:
      kafka:
        condition: service_healthy
    restart: unless-stopped
    command: python social-media-web-scrapping/gold_layer.py
  
  # # Pathway Vector Store Pipeline (replaces Qdrant)
  pathway-vector-pipeline:
    build:
      context: ./social-media-web-scrapping
      dockerfile: Dockerfile.vector
    container_name: pathway-vector-ingestion-pipeline
    networks:
      - kafka-network
    ports:
      - "8766:8766"
    environment:
      - KAFKA_BROKER=kafka:9092
      - GOLD_SECTOR_TOPIC=gold_sector_analytics
      - VECTOR_STORE_HOST=0.0.0.0
      - VECTOR_STORE_PORT=8766
      - EMBEDDING_MODEL=all-MiniLM-L6-v2
      - CACHE_DIR=/app/cache
    volumes:
      - ./:/app:cached,z
      - ./social-media-web-scrapping/pathway-cache:/app/cache
    depends_on:
      kafka:
        condition: service_healthy
      gold-layer:
        condition: service_started
    restart: unless-stopped
    command: python social-media-web-scrapping/pathway_vector_pipeline.py
    healthcheck:
      test:
        ["CMD-SHELL", "curl -f http://localhost:8766/v1/statistics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  client-backend:
    build:
      context: ./pathway-be
    env_file:
      - .env
    environment:
      PORT: 4000
      NODE_ENV: production
      REDIS_HOST_URL: "redis://redis:6379/"
      KAFKA_URL: ${KAFKA_URL}
      BACKTEST_REMOTE: ${BACKTEST_REMOTE}
      DEVELOPER_BACKTEST_URL: ${DEVELOPER_BACKTEST_URL}
      BACKEND_URL: ${BACKEND_URL}
      FRONTEND_URL: ${FRONTEND_URL}
      BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET}
      GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID}
      GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET}
      CHATBOT_SECRET_KEY: ${CHATBOT_SECRET_KEY}
      MONGO_BACKEND_URL: ${MONGO_BACKEND_URL}
      MONGO_HISTORIAL_DATA_URL: ${MONGO_HISTORIAL_DATA_URL}
      SMTP_HOST: ${SMTP_HOST}
      SMTP_PORT: ${SMTP_PORT}
      SMTP_USER: ${SMTP_USER}
      SMTP_PASS: ${SMTP_PASS}
      SMTP_FROM: ${SMTP_FROM}
    ports:
      - "4000:4000"
    restart: unless-stopped
    networks:
      - backend
      - kafka-network
      - observability-network
    depends_on:
      kafka:
        condition: service_healthy

  # stock-repository-job:
  #   build: ./stock-repository-job
  #   environment:
  #     KITE_API_KEY: ${KITE_API_KEY}
  #     KITE_ACCESS_TOKEN: ${KITE_ACCESS_TOKEN}
  #     HIST_MONGODB_URI: ${HIST_MONGODB_URI}
  #   networks:
  #     - backend

  nginx:
    # image: helix38/custom-nginx:latest
    build:
      context: ./nginx
    depends_on:
      - client-backend
      - kafka
      - grafana
      - prometheus
    ports:
      - "80:80"
    networks:
      - observability-network
      - kafka-network
      - backend