docker-compose-dev.yml

# this docker-compose file is used for local development and testing
services:
  # KAFKA
  kafka:
    image: apache/kafka:3.7.2
    container_name: kafka
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_NUM_PARTITIONS: 3
    ports:
      - 9092:9092
    healthcheck:
      test:
        [
          "CMD",
          "./opt/kafka/bin/kafka-topics.sh",
          "--list",
          "--bootstrap-server",
          "localhost:9092"
        ]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - botdetector-network

  kafka_setup:
    container_name: kafka_setup
    image: bd/kafka_setup
    build:
      context: ./_kafka
    environment:
      - KAFKA_BROKER=kafka:9092
    networks:
      - botdetector-network
    depends_on:
      kafka:
        condition: service_healthy

  kafdrop:
    container_name: kafdrop
    image: obsidiandynamics/kafdrop:latest
    environment:
      - KAFKA_BROKERCONNECT=kafka:9092
      - JVM_OPTS=-Xms32M -Xmx64M
      - SERVER_SERVLET_CONTEXTPATH=/
    ports:
      - 9000:9000
    restart: on-failure
    networks:
      - botdetector-network
    depends_on:
      kafka:
        condition: service_healthy

  # MYSQL
  mysql:
    container_name: mysql
    image: bd/mysql
    build:
      context: ./_mysql
    environment:
      - MYSQL_ROOT_PASSWORD=root_bot_buster
    ports:
      - 3307:3306
    networks:
      - botdetector-network
    healthcheck:
      test: ["CMD-SHELL", "mysqladmin ping -h localhost -u root -proot_bot_buster"]
      interval: 10s
      retries: 3
      start_period: 30s
      timeout: 5s

  mysql_setup:
    container_name: mysql_setup
    image: bd/mysql_setup
    build:
      context: ./_mysql_data
    environment:
      - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata
      - DEBUG=False
    networks:
      - botdetector-network
    depends_on:
      mysql:
        condition: service_healthy

  # MINIO
  minio:
    image: minio/minio
    container_name: minio
    expose:
      - "9000"
      - "9001"
    ports:
      - 9001
    environment:
      MINIO_ROOT_USER: "minio_user"
      MINIO_ROOT_PASSWORD: "minio_password"
    volumes:
      - ./_minio_data:/data
    healthcheck:
      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
      interval: 1s
      timeout: 10s
      retries: 5
    command: server /data --console-address ":9001"
    networks:
      - botdetector-network

  # Create a bucket named "bucket" if it doesn't exist
  minio-create-bucket:
    image: minio/mc
    container_name: minio_create_bucket
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      bash -c "
      mc alias set minio http://minio:9000 minio_user minio_password &&
      if ! mc ls minio/bucket; then
        mc mb minio/models
      else
        echo 'bucket already exists'
      fi
      "

  # COMPONENTS
  hiscore_scraper:
    container_name: hiscore_scraper
    image: bd/hiscore_scraper:dev
    build:
      context: ./
      dockerfile: ./projects/hiscore_scraper/Dockerfile
      target: dev
    networks:
      - botdetector-network
    environment:
      - DEBUG=1
    env_file:
      - .env
    depends_on:
      kafka_setup:
        condition: service_completed_successfully

  runemetrics_scraper:
    container_name: runemetrics_scraper
    image: bd/runemetrics_scraper:dev
    build:
      context: ./
      dockerfile: ./projects/runemetrics_scraper/Dockerfile
      target: dev
    networks:
      - botdetector-network
    env_file:
      - .env
    depends_on:
      kafka_setup:
        condition: service_completed_successfully

  worker_hiscore:
    container_name: worker_hiscore
    image: bd/worker_hiscore:dev
    build:
      context: ./
      dockerfile: ./projects/worker_hiscore/Dockerfile
      target: dev
    networks:
      - botdetector-network
    env_file:
      - .env
    depends_on:
      kafka:
        condition: service_healthy
      kafka_setup:
        condition: service_completed_successfully
      mysql:
        condition: service_healthy
      mysql_setup:
        condition: service_completed_successfully

  worker_report:
    container_name: worker_report
    image: bd/worker_report:dev
    build:
      context: ./
      dockerfile: ./projects/worker_report/Dockerfile
      target: dev
    networks:
      - botdetector-network
    env_file:
      - .env
    depends_on:
      kafka_setup:
        condition: service_completed_successfully
      mysql_setup:
        condition: service_completed_successfully

  worker_ml:
    container_name: worker_ml
    image: bd/worker_ml:dev
    build:
      context: ./
      dockerfile: ./projects/worker_ml/Dockerfile
      target: dev
    environment:
      - DATABASE_URL=mysql+asyncmy://ml-worker:ml_worker_pw@mysql:3306/playerdata
      - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
      - BASE_URL=http://api_ml:5000
      - MAX_MESSAGES=10
      - MAX_INTERVAL_MS=5000
      - MODEL_NAME=multi_model_v1
    volumes:
      - ./bases:/project/bases
      - ./components:/project/components
    networks:
      - botdetector-network
    depends_on:
      kafka_setup:
        condition: service_completed_successfully
      mysql_setup:
        condition: service_completed_successfully
      api_ml:
        condition: service_healthy

  scrape_task_producer:
    container_name: scrape_task_producer
    image: bd/scrape_task_producer:dev
    build:
      context: ./
      dockerfile: ./projects/scrape_task_producer/Dockerfile
      target: dev
    networks:
      - botdetector-network
    env_file:
      - .env
    depends_on:
      kafka_setup:
        condition: service_completed_successfully
      mysql_setup:
        condition: service_completed_successfully

  job_prune_hs_data:
    container_name: job_prune_hs_data
    image: bd/job_prune_hs_data:dev
    build:
      context: ./
      dockerfile: ./projects/job_prune_hs_data/Dockerfile
      target: dev
    networks:
      - botdetector-network
    environment:
      - LIMIT=100
      - DATABASE_URL=mysql+asyncmy://job-prune-hs:job_prune_hs_pw@mysql:3306/playerdata
    env_file:
      - .env
    depends_on:
      mysql_setup:
        condition: service_completed_successfully

  api_public:
    container_name: api_public
    image: bd/api_public:dev
    build:
      context: .
      dockerfile: ./projects/api_public/Dockerfile
      target: dev
    env_file:
      - .env
    environment:
      - KAFKA_HOST=kafka:9092
      - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata
      - ENV=DEV
      - POOL_RECYCLE=60
      - POOL_TIMEOUT=30
    volumes:
      - ./bases:/project/bases
      - ./components:/project/components
    ports:
      - 5000
    networks:
      - botdetector-network
    depends_on:
      kafka_setup:
        condition: service_completed_successfully
      mysql_setup:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s

  website:
    container_name: website
    image: bd/website:dev
    build:
      context: .
      dockerfile: ./projects/website/Dockerfile
      target: dev
    env_file:
      - .env
    environment:
      BD_TOKEN: ""
      ENV: "DEV"
      RELEASE_VERSION: "0.1"
      PATREON_CLIENT_ID: ""
      PATREON_CLIENT_SECRET: ""
    volumes:
      - ./bases:/project/bases
      - ./components:/project/components
    ports:
      - 5000
    networks:
      - botdetector-network
    healthcheck:
      test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/monitoring\").getcode()==200 else 1)'"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
  api_ml:
    container_name: api_ml
    image: bd/api_ml:dev
    build:
      context: .
      dockerfile: ./projects/api_ml/Dockerfile
      target: dev
    # command: ["sleep", "infinity"]
    env_file:
      - .env
    environment:
      - MLFLOW_S3_ENDPOINT_URL=http://minio:9000
      - AWS_ACCESS_KEY_ID=minio_user
      - AWS_SECRET_ACCESS_KEY=minio_password
      - UV_HTTP_TIMEOUT=120
    networks:
      - botdetector-network
    depends_on:
      minio:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s

networks:
  botdetector-network:
    name: bd-network