diff --git a/README.md b/README.md index 88da561..c889a26 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,12 @@ flowchart TD ```sh find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 Updating lock in $(pwd)"; uv lock' \; ``` -# syncing in all directories, so uv cache is setup ```sh find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 syncing in $(pwd)"; uv sync' \; +``` +```sh +find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 Removing venv in $(pwd)"; rm -rf .venv/' \; +``` +```sh +find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 Removing Dockerfile.bak in $(pwd)"; rm Dockerfile.bak' \; ``` \ No newline at end of file diff --git a/bases/bot_detector/worker_report/main.py b/bases/bot_detector/worker_report/main.py index 5d66e1b..3040948 100644 --- a/bases/bot_detector/worker_report/main.py +++ b/bases/bot_detector/worker_report/main.py @@ -138,6 +138,7 @@ async def main(): ## producer report_producer = RepoReportsToInsertProducer( bootstrap_servers=b_server, + max_async_calls=100, ) # start kafka producers and consumers diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 8d44d9f..2554517 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -1,3 +1,4 @@ +# this docker-compose file is used for debugging and development purposes services: kafka: image: apache/kafka:3.7.2 diff --git a/docker-compose-prod.yml b/docker-compose-dev.yml similarity index 73% rename from docker-compose-prod.yml rename to docker-compose-dev.yml index 92c754b..debafca 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-dev.yml @@ -1,3 +1,4 @@ +# this docker-compose file is used for local development and testing services: # KAFKA kafka: @@ -32,12 +33,12 @@ services: retries: 5 networks: - botdetector-network + kafka_setup: container_name: kafka_setup - image: bd/kafka_setup # tags the image if build + image: bd/kafka_setup build: context: ./_kafka - # command: ["sleep", "infinity"] environment: - KAFKA_BROKER=kafka:9092 networks: @@ -45,6 +46,7 @@ services: depends_on: kafka: condition: service_healthy + kafdrop: container_name: kafdrop image: obsidiandynamics/kafdrop:latest @@ -53,22 +55,22 @@ services: - JVM_OPTS=-Xms32M -Xmx64M - SERVER_SERVLET_CONTEXTPATH=/ ports: - - 9042:9000 # too many things use 9000 like portainer.. lets use 9042 instead? + - 9042:9000 restart: on-failure networks: - botdetector-network depends_on: kafka: condition: service_healthy + # MYSQL mysql: container_name: mysql - image: bd/mysql # tags the image if build + image: bd/mysql build: context: ./_mysql environment: - MYSQL_ROOT_PASSWORD=root_bot_buster - # - MYSQL_DATABASE=playerdata ports: - 3307:3306 networks: @@ -79,12 +81,12 @@ services: retries: 3 start_period: 30s timeout: 5s + mysql_setup: container_name: mysql_setup - image: bd/mysql_setup # tags the image if build + image: bd/mysql_setup build: context: ./_mysql_data - # command: ["sleep", "infinity"] environment: - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - DEBUG=False @@ -93,9 +95,11 @@ services: depends_on: mysql: condition: service_healthy + # MINIO minio: image: minio/minio + container_name: minio expose: - "9000" - "9001" @@ -114,9 +118,11 @@ services: command: server /data --console-address ":9001" networks: - botdetector-network + # Create a bucket named "bucket" if it doesn't exist minio-create-bucket: image: minio/mc + container_name: minio_create_bucket depends_on: minio: condition: service_healthy @@ -129,57 +135,49 @@ services: echo 'bucket already exists' fi " + # COMPONENTS hiscore_scraper: container_name: hiscore_scraper - image: bd/hiscore_scraper # tags the image if build + image: bd/hiscore_scraper:dev build: context: ./ dockerfile: ./projects/hiscore_scraper/Dockerfile - target: production - # command: ["uv", "run", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + target: dev networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully + runemetrics_scraper: container_name: runemetrics_scraper - image: bd/runemetrics_scraper # tags the image if build + image: bd/runemetrics_scraper:dev build: context: ./ dockerfile: ./projects/runemetrics_scraper/Dockerfile - target: production - # command: ["sleep", "infinity"] - # command: ["uv", "run", "bot_detector/runemetrics_scraper/src/core.py"] + target: dev networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully + worker_hiscore: container_name: worker_hiscore - image: bd/worker_hiscore # tags the image if build + image: bd/worker_hiscore:dev build: context: ./ dockerfile: ./projects/worker_hiscore/Dockerfile - target: production - # command: ["sleep", "infinity"] - # command: ["uv", "run", "bot_detector/worker_hiscore/src/core.py"] + target: dev networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka: condition: service_healthy @@ -189,33 +187,31 @@ services: condition: service_healthy mysql_setup: condition: service_completed_successfully + worker_report: container_name: worker_report - image: bd/worker_report # tags the image if build + image: bd/worker_report:dev build: context: ./ dockerfile: ./projects/worker_report/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: dev networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully mysql_setup: condition: service_completed_successfully + worker_ml: container_name: worker_ml - image: bd/worker_ml # tags the image if build + image: bd/worker_ml:dev build: context: ./ dockerfile: ./projects/worker_ml/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: dev environment: - DATABASE_URL=mysql+asyncmy://ml-worker:ml_worker_pw@mysql:3306/playerdata - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 @@ -223,10 +219,11 @@ services: - MAX_MESSAGES=10 - MAX_INTERVAL_MS=5000 - MODEL_NAME=multi_model_v1 + volumes: + - ./bases:/project/bases + - ./components:/project/components networks: - botdetector-network - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully @@ -234,34 +231,31 @@ services: condition: service_completed_successfully api_ml: condition: service_healthy - + scrape_task_producer: container_name: scrape_task_producer - image: bd/scrape_task_producer # tags the image if build + image: bd/scrape_task_producer:dev build: context: ./ dockerfile: ./projects/scrape_task_producer/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: dev networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully mysql_setup: condition: service_completed_successfully + job_prune_hs_data: container_name: job_prune_hs_data - image: bd/job_prune_hs_data # tags the image if build + image: bd/job_prune_hs_data:dev build: context: ./ dockerfile: ./projects/job_prune_hs_data/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: dev networks: - botdetector-network environment: @@ -269,19 +263,17 @@ services: - DATABASE_URL=mysql+asyncmy://job-prune-hs:job_prune_hs_pw@mysql:3306/playerdata env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: mysql_setup: condition: service_completed_successfully + api_public: container_name: api_public - image: bd/api_public:prod + image: bd/api_public:dev build: context: . dockerfile: ./projects/api_public/Dockerfile - target: prod - # command: ["sleep", "infinity"] + target: dev env_file: - .env environment: @@ -291,8 +283,8 @@ services: - POOL_RECYCLE=60 - POOL_TIMEOUT=30 volumes: - - ./bases:/app/bases - - ./components:/app/components + - ./bases:/project/bases + - ./components:/project/components ports: - 5000 networks: @@ -308,21 +300,14 @@ services: timeout: 5s retries: 3 start_period: 30s - wait_for_api_public: - image: alpine:latest - container_name: wait_for_api_public - command: ["sh", "-c", "echo 'api_public healthy'"] - depends_on: - api_public: - condition: service_healthy - networks: - - botdetector-network + website: container_name: website - image: bd/website + image: bd/website:dev build: context: . dockerfile: ./projects/website/Dockerfile + target: dev env_file: - .env environment: @@ -332,7 +317,8 @@ services: PATREON_CLIENT_ID: "" PATREON_CLIENT_SECRET: "" volumes: - - uv_cache:/root/.cache/uv + - ./bases:/project/bases + - ./components:/project/components ports: - 5000 networks: @@ -345,14 +331,12 @@ services: start_period: 30s api_ml: container_name: api_ml - image: bd/api_ml + image: bd/api_ml:dev build: context: . dockerfile: ./projects/api_ml/Dockerfile - target: builder + target: dev # command: ["sleep", "infinity"] - command: > - sh -c "cd ../.. && projects/api_ml/.venv/bin/uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir bases/bot_detector/api_ml" env_file: - .env environment: @@ -360,12 +344,6 @@ services: - AWS_ACCESS_KEY_ID=minio_user - AWS_SECRET_ACCESS_KEY=minio_password - UV_HTTP_TIMEOUT=120 - volumes: - - ./bases:/app/bases - - ./components:/app/components - - uv_cache:/root/.cache/uv - ports: - - 5001:5000 networks: - botdetector-network depends_on: @@ -380,16 +358,4 @@ services: networks: botdetector-network: - name: bd-network - -# To create the docker volume for uv cache (run only once) -# docker volume create uv_cache - -# To copy existing cache from host to docker volume (run only once) -# docker run --rm \ -# -v uv_cache:/data \ -# -v ~/.cache/uv:/source \ -# alpine sh -c "cp -r /source/* /data/" -volumes: - uv_cache: - external: true \ No newline at end of file + name: bd-network \ No newline at end of file diff --git a/docker-compose-oc.yml b/docker-compose-oc.yml index 32fc596..8dc30e2 100644 --- a/docker-compose-oc.yml +++ b/docker-compose-oc.yml @@ -27,8 +27,9 @@ services: - ./_mysql:/project/_mysql # configs and docs - ./docker-compose.yml:/project/docker-compose.yml - - ./docker-compose-dev.yml:/project/docker-compose-dev.yml - ./docker-compose-oc.yml:/project/docker-compose-oc.yml + - ./docker-compose-dev.yml:/project/docker-compose-dev.yml + - ./docker-compose-debug.yml:/project/docker-compose-debug.yml - ./README.md:/project/README.md env_file: - ./_opencode/.env diff --git a/docker-compose.yml b/docker-compose.yml index 56b85bb..3b6c5c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +# this docker-compose file is used for production deployment services: # KAFKA kafka: @@ -32,12 +33,12 @@ services: retries: 5 networks: - botdetector-network + kafka_setup: container_name: kafka_setup - image: bd/kafka_setup # tags the image if build + image: bd/kafka_setup build: context: ./_kafka - # command: ["sleep", "infinity"] environment: - KAFKA_BROKER=kafka:9092 networks: @@ -45,6 +46,7 @@ services: depends_on: kafka: condition: service_healthy + kafdrop: container_name: kafdrop image: obsidiandynamics/kafdrop:latest @@ -53,22 +55,22 @@ services: - JVM_OPTS=-Xms32M -Xmx64M - SERVER_SERVLET_CONTEXTPATH=/ ports: - - 9042:9000 # too many things use 9000 like portainer.. lets use 9042 instead? + - 9042:9000 restart: on-failure networks: - botdetector-network depends_on: kafka: condition: service_healthy + # MYSQL mysql: container_name: mysql - image: bd/mysql # tags the image if build + image: bd/mysql build: context: ./_mysql environment: - MYSQL_ROOT_PASSWORD=root_bot_buster - # - MYSQL_DATABASE=playerdata ports: - 3307:3306 networks: @@ -79,12 +81,12 @@ services: retries: 3 start_period: 30s timeout: 5s + mysql_setup: container_name: mysql_setup - image: bd/mysql_setup # tags the image if build + image: bd/mysql_setup build: context: ./_mysql_data - # command: ["sleep", "infinity"] environment: - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - DEBUG=False @@ -93,6 +95,7 @@ services: depends_on: mysql: condition: service_healthy + # MINIO minio: image: minio/minio @@ -115,6 +118,7 @@ services: command: server /data --console-address ":9001" networks: - botdetector-network + # Create a bucket named "bucket" if it doesn't exist minio-create-bucket: image: minio/mc @@ -131,57 +135,51 @@ services: echo 'bucket already exists' fi " + networks: + - botdetector-network + # COMPONENTS hiscore_scraper: container_name: hiscore_scraper - image: bd/hiscore_scraper # tags the image if build + image: bd/hiscore_scraper:prd build: context: ./ dockerfile: ./projects/hiscore_scraper/Dockerfile - target: production - # command: ["uv", "run", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + target: prod networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully + runemetrics_scraper: container_name: runemetrics_scraper - image: bd/runemetrics_scraper # tags the image if build + image: bd/runemetrics_scraper:prd build: context: ./ dockerfile: ./projects/runemetrics_scraper/Dockerfile - target: production - # command: ["sleep", "infinity"] - # command: ["uv", "run", "bot_detector/runemetrics_scraper/src/core.py"] + target: prod networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully + worker_hiscore: container_name: worker_hiscore - image: bd/worker_hiscore # tags the image if build + image: bd/worker_hiscore:prd build: context: ./ dockerfile: ./projects/worker_hiscore/Dockerfile - target: production - # command: ["sleep", "infinity"] - # command: ["uv", "run", "bot_detector/worker_hiscore/src/core.py"] + target: prod networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka: condition: service_healthy @@ -191,33 +189,31 @@ services: condition: service_healthy mysql_setup: condition: service_completed_successfully + worker_report: container_name: worker_report - image: bd/worker_report # tags the image if build + image: bd/worker_report:prd build: context: ./ dockerfile: ./projects/worker_report/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: prod networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully mysql_setup: condition: service_completed_successfully + worker_ml: container_name: worker_ml - image: bd/worker_ml # tags the image if build + image: bd/worker_ml:prd build: context: ./ dockerfile: ./projects/worker_ml/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: prod environment: - DATABASE_URL=mysql+asyncmy://ml-worker:ml_worker_pw@mysql:3306/playerdata - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 @@ -227,8 +223,6 @@ services: - MODEL_NAME=multi_model_v1 networks: - botdetector-network - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully @@ -236,34 +230,31 @@ services: condition: service_completed_successfully api_ml: condition: service_healthy - + scrape_task_producer: container_name: scrape_task_producer - image: bd/scrape_task_producer # tags the image if build + image: bd/scrape_task_producer:prd build: context: ./ dockerfile: ./projects/scrape_task_producer/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: prod networks: - botdetector-network env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: kafka_setup: condition: service_completed_successfully mysql_setup: condition: service_completed_successfully + job_prune_hs_data: container_name: job_prune_hs_data - image: bd/job_prune_hs_data # tags the image if build + image: bd/job_prune_hs_data:prd build: context: ./ dockerfile: ./projects/job_prune_hs_data/Dockerfile - target: production - # command: ["sleep", "infinity"] + target: prod networks: - botdetector-network environment: @@ -271,19 +262,17 @@ services: - DATABASE_URL=mysql+asyncmy://job-prune-hs:job_prune_hs_pw@mysql:3306/playerdata env_file: - .env - volumes: - - uv_cache:/root/.cache/uv depends_on: mysql_setup: condition: service_completed_successfully + api_public: container_name: api_public - image: bd/api_public:dev + image: bd/api_public:prd build: context: . dockerfile: ./projects/api_public/Dockerfile - target: dev - # command: ["sleep", "infinity"] + target: prod env_file: - .env environment: @@ -292,10 +281,6 @@ services: - ENV=DEV - POOL_RECYCLE=60 - POOL_TIMEOUT=30 - volumes: - - ./bases:/bases - - ./components:/components - # - uv_cache:/root/.cache/uv ports: - 5000 networks: @@ -311,21 +296,14 @@ services: timeout: 5s retries: 3 start_period: 30s - wait_for_api_public: - image: alpine:latest - container_name: wait_for_api_public - command: ["sh", "-c", "echo 'api_public healthy'"] - depends_on: - api_public: - condition: service_healthy - networks: - - botdetector-network + website: container_name: website - image: bd/website + image: bd/website:prd build: context: . dockerfile: ./projects/website/Dockerfile + target: prod env_file: - .env environment: @@ -334,10 +312,6 @@ services: RELEASE_VERSION: "0.1" PATREON_CLIENT_ID: "" PATREON_CLIENT_SECRET: "" - volumes: - - uv_cache:/root/.cache/uv - ports: - - 5000 networks: - botdetector-network healthcheck: @@ -346,14 +320,14 @@ services: timeout: 5s retries: 3 start_period: 30s + api_ml: container_name: api_ml - image: bd/api_ml + image: bd/api_ml:prd build: context: . dockerfile: ./projects/api_ml/Dockerfile - target: dev - # command: ["sleep", "infinity"] + target: prod env_file: - .env environment: @@ -361,12 +335,8 @@ services: - AWS_ACCESS_KEY_ID=minio_user - AWS_SECRET_ACCESS_KEY=minio_password - UV_HTTP_TIMEOUT=120 - volumes: - - ./bases:/app/bases - - ./components:/app/components - - uv_cache:/root/.cache/uv ports: - - 5001:5000 + - 5000 networks: - botdetector-network depends_on: @@ -381,16 +351,4 @@ services: networks: botdetector-network: - name: bd-network - -# To create the docker volume for uv cache (run only once) -# docker volume create uv_cache - -# To copy existing cache from host to docker volume (run only once) -# docker run --rm \ -# -v uv_cache:/data \ -# -v ~/.cache/uv:/source \ -# alpine sh -c "cp -r /source/* /data/" -volumes: - uv_cache: - external: true \ No newline at end of file + name: bd-network \ No newline at end of file diff --git a/projects/api_ml/Dockerfile b/projects/api_ml/Dockerfile index bf9e21e..7cc294a 100644 --- a/projects/api_ml/Dockerfile +++ b/projects/api_ml/Dockerfile @@ -1,57 +1,78 @@ +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ FROM python:3.12-slim-bookworm AS base -# Python optimizations ENV PYTHONUNBUFFERED=1 ENV UV_COMPILE_BYTECODE=1 -WORKDIR /app +WORKDIR /project -FROM base AS builder -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /bin/ -WORKDIR /app +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder -COPY ./projects/api_ml ./projects/api_ml +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv COPY ./bases ./bases COPY ./components ./components +COPY ./projects/api_ml ./projects/api_ml +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/api_ml RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --project ./projects/api_ml + uv sync --frozen --no-dev -FROM base AS dev -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv uvx/ /bin/ -WORKDIR /app +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -COPY --from=builder /app/projects/api_ml/.venv /app/.venv +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -ENV PATH="/app/.venv/bin:$PATH" -ENV PYTHONPATH="/app" +ENV PYTHONPATH="/project" -COPY ./projects/api_ml ./projects/api_ml COPY ./bases ./bases COPY ./components ./components +COPY ./projects/api_ml ./projects/api_ml +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/api_ml -# this installs dev dependencies RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ - uv sync --frozen --project ./projects/api_ml \ - && cp -r projects/api_ml/.venv/ .venv/ + uv sync --dev + +ENV PATH="/project/projects/api_ml/.venv/bin:$PATH" CMD ["uvicorn", "bot_detector.api_ml.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--reload"] -FROM base AS prod -WORKDIR /app +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Create a secure user +# Non-root user RUN addgroup --system appuser && \ - adduser --system --ingroup appuser --home /app --no-create-home --disabled-password --uid 5678 appuser + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser + +# SAME PATH as builder +WORKDIR /project/projects/api_ml -COPY --from=builder --chown=appuser /app/projects/api_ml/.venv /app/.venv +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/api_ml/.venv \ + /project/projects/api_ml/.venv -ENV PATH="/app/.venv/bin:$PATH" -ENV PYTHONPATH="/app" +ENV PATH="/project/projects/api_ml/.venv/bin:$PATH" USER appuser -CMD ["python3", "-m", "uvicorn", "bot_detector.api_ml.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD ["uvicorn", "bot_detector.api_ml.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index ece9e6b..8185200 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -1,57 +1,78 @@ +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ FROM python:3.12-slim-bookworm AS base -# Python optimizations ENV PYTHONUNBUFFERED=1 ENV UV_COMPILE_BYTECODE=1 -WORKDIR /app +WORKDIR /project -FROM base AS builder -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /bin/ -WORKDIR /app +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder -COPY ./projects/api_public ./projects/api_public +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv COPY ./bases ./bases COPY ./components ./components +COPY ./projects/api_public ./projects/api_public +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/api_public RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --project ./projects/api_public + uv sync --frozen --no-dev -FROM base AS dev -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv uvx/ /bin/ -WORKDIR /app +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -COPY --from=builder /app/projects/api_public/.venv /app/.venv +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -ENV PATH="/app/.venv/bin:$PATH" -ENV PYTHONPATH="/app" +ENV PYTHONPATH="/project" -COPY ./projects/api_public ./projects/api_public COPY ./bases ./bases COPY ./components ./components +COPY ./projects/api_public ./projects/api_public +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/api_public -# this installs dev dependencies RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ - uv sync --frozen --project ./projects/api_public \ - && cp -r projects/api_public/.venv/ .venv/ + uv sync --dev + +ENV PATH="/project/projects/api_public/.venv/bin:$PATH" CMD ["uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--reload"] -FROM base AS prod -WORKDIR /app +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Create a secure user +# Non-root user RUN addgroup --system appuser && \ - adduser --system --ingroup appuser --home /app --no-create-home --disabled-password --uid 5678 appuser + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser + +# SAME PATH as builder +WORKDIR /project/projects/api_public -COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/.venv +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/api_public/.venv \ + /project/projects/api_public/.venv -ENV PATH="/app/.venv/bin:$PATH" -ENV PYTHONPATH="/app" +ENV PATH="/project/projects/api_public/.venv/bin:$PATH" USER appuser -CMD ["python3", "-m", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD ["uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/hiscore_scraper/Dockerfile b/projects/hiscore_scraper/Dockerfile index 97b14ba..85f35a5 100644 --- a/projects/hiscore_scraper/Dockerfile +++ b/projects/hiscore_scraper/Dockerfile @@ -1,30 +1,81 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: runtime-only venv (path is FINAL) +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/hiscore_scraper/Dockerfile ./projects/hiscore_scraper/Dockerfile +COPY ./projects/hiscore_scraper/pyproject.toml ./projects/hiscore_scraper/pyproject.toml +COPY ./projects/hiscore_scraper/uv.lock ./projects/hiscore_scraper/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/hiscore_scraper + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +# ------------------------------------------------------------ +# Dev: full workspace + dev deps (own venv) +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/hiscore_scraper/Dockerfile ./projects/hiscore_scraper/Dockerfile +COPY ./projects/hiscore_scraper/pyproject.toml ./projects/hiscore_scraper/pyproject.toml +COPY ./projects/hiscore_scraper/uv.lock ./projects/hiscore_scraper/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/hiscore_scraper + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/hiscore_scraper/.venv/bin:$PATH" + +CMD ["hiscore_scraper"] -WORKDIR /app/projects/hiscore_scraper -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM base AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/hiscore_scraper -WORKDIR /app/projects/hiscore_scraper +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/hiscore_scraper/.venv \ + /project/projects/hiscore_scraper/.venv -COPY --from=builder --chown=appuser /app/projects/hiscore_scraper/.venv /app/projects/hiscore_scraper/.venv +ENV PATH="/project/projects/hiscore_scraper/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/hiscore_scraper"] +CMD ["hiscore_scraper"] diff --git a/projects/job_hs_migration_v3/Dockerfile b/projects/job_hs_migration_v3/Dockerfile index a2d128b..f4cad12 100644 --- a/projects/job_hs_migration_v3/Dockerfile +++ b/projects/job_hs_migration_v3/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/job_hs_migration_v3/Dockerfile ./projects/job_hs_migration_v3/Dockerfile +COPY ./projects/job_hs_migration_v3/pyproject.toml ./projects/job_hs_migration_v3/pyproject.toml +COPY ./projects/job_hs_migration_v3/uv.lock ./projects/job_hs_migration_v3/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/job_hs_migration_v3 + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/job_hs_migration_v3/Dockerfile ./projects/job_hs_migration_v3/Dockerfile +COPY ./projects/job_hs_migration_v3/pyproject.toml ./projects/job_hs_migration_v3/pyproject.toml +COPY ./projects/job_hs_migration_v3/uv.lock ./projects/job_hs_migration_v3/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/job_hs_migration_v3 + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/job_hs_migration_v3/.venv/bin:$PATH" + +CMD ["job_hs_migration_v3"] -WORKDIR /app/projects/job_hs_migration_v3 -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/job_hs_migration_v3 -WORKDIR /app/projects/job_hs_migration_v3 +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/job_hs_migration_v3/.venv \ + /project/projects/job_hs_migration_v3/.venv -COPY --from=builder --chown=appuser /app/projects/job_hs_migration_v3/.venv /app/projects/job_hs_migration_v3/.venv +ENV PATH="/project/projects/job_hs_migration_v3/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/job_hs_migration_v3"] +CMD ["job_hs_migration_v3"] diff --git a/projects/job_prune_hs_data/Dockerfile b/projects/job_prune_hs_data/Dockerfile index 6be8702..930ca88 100644 --- a/projects/job_prune_hs_data/Dockerfile +++ b/projects/job_prune_hs_data/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/job_prune_hs_data/Dockerfile ./projects/job_prune_hs_data/Dockerfile +COPY ./projects/job_prune_hs_data/pyproject.toml ./projects/job_prune_hs_data/pyproject.toml +COPY ./projects/job_prune_hs_data/uv.lock ./projects/job_prune_hs_data/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/job_prune_hs_data + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/job_prune_hs_data/Dockerfile ./projects/job_prune_hs_data/Dockerfile +COPY ./projects/job_prune_hs_data/pyproject.toml ./projects/job_prune_hs_data/pyproject.toml +COPY ./projects/job_prune_hs_data/uv.lock ./projects/job_prune_hs_data/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/job_prune_hs_data + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/job_prune_hs_data/.venv/bin:$PATH" + +CMD ["job_prune_hs_data"] -WORKDIR /app/projects/job_prune_hs_data -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/job_prune_hs_data -WORKDIR /app/projects/job_prune_hs_data +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/job_prune_hs_data/.venv \ + /project/projects/job_prune_hs_data/.venv -COPY --from=builder --chown=appuser /app/projects/job_prune_hs_data/.venv /app/projects/job_prune_hs_data/.venv +ENV PATH="/project/projects/job_prune_hs_data/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/job_prune_hs_data"] +CMD ["job_prune_hs_data"] diff --git a/projects/runemetrics_scraper/Dockerfile b/projects/runemetrics_scraper/Dockerfile index 2b14bd8..eab791b 100644 --- a/projects/runemetrics_scraper/Dockerfile +++ b/projects/runemetrics_scraper/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/runemetrics_scraper/Dockerfile ./projects/runemetrics_scraper/Dockerfile +COPY ./projects/runemetrics_scraper/pyproject.toml ./projects/runemetrics_scraper/pyproject.toml +COPY ./projects/runemetrics_scraper/uv.lock ./projects/runemetrics_scraper/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/runemetrics_scraper + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/runemetrics_scraper/Dockerfile ./projects/runemetrics_scraper/Dockerfile +COPY ./projects/runemetrics_scraper/pyproject.toml ./projects/runemetrics_scraper/pyproject.toml +COPY ./projects/runemetrics_scraper/uv.lock ./projects/runemetrics_scraper/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/runemetrics_scraper + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/runemetrics_scraper/.venv/bin:$PATH" + +CMD ["runemetrics_scraper"] -WORKDIR /app/projects/runemetrics_scraper -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/runemetrics_scraper -WORKDIR /app/projects/runemetrics_scraper +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/runemetrics_scraper/.venv \ + /project/projects/runemetrics_scraper/.venv -COPY --from=builder --chown=appuser /app/projects/runemetrics_scraper/.venv /app/projects/runemetrics_scraper/.venv +ENV PATH="/project/projects/runemetrics_scraper/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/runemetrics_scraper"] +CMD ["runemetrics_scraper"] diff --git a/projects/scrape_task_producer/Dockerfile b/projects/scrape_task_producer/Dockerfile index cf5bbc0..e82edab 100644 --- a/projects/scrape_task_producer/Dockerfile +++ b/projects/scrape_task_producer/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/scrape_task_producer/Dockerfile ./projects/scrape_task_producer/Dockerfile +COPY ./projects/scrape_task_producer/pyproject.toml ./projects/scrape_task_producer/pyproject.toml +COPY ./projects/scrape_task_producer/uv.lock ./projects/scrape_task_producer/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/scrape_task_producer + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/scrape_task_producer/Dockerfile ./projects/scrape_task_producer/Dockerfile +COPY ./projects/scrape_task_producer/pyproject.toml ./projects/scrape_task_producer/pyproject.toml +COPY ./projects/scrape_task_producer/uv.lock ./projects/scrape_task_producer/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/scrape_task_producer + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/scrape_task_producer/.venv/bin:$PATH" + +CMD ["scrape_task_producer"] -WORKDIR /app/projects/scrape_task_producer -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/scrape_task_producer -WORKDIR /app/projects/scrape_task_producer +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/scrape_task_producer/.venv \ + /project/projects/scrape_task_producer/.venv -COPY --from=builder --chown=appuser /app/projects/scrape_task_producer/.venv /app/projects/scrape_task_producer/.venv +ENV PATH="/project/projects/scrape_task_producer/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/scrape_task_producer"] +CMD ["scrape_task_producer"] diff --git a/projects/website/Dockerfile b/projects/website/Dockerfile index 211b76e..83e3358 100644 --- a/projects/website/Dockerfile +++ b/projects/website/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base -# Copy uv from external repository -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /uvx /bin/ +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 -# set the working directory -WORKDIR /app +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/website/Dockerfile ./projects/website/Dockerfile +COPY ./projects/website/pyproject.toml ./projects/website/pyproject.toml +COPY ./projects/website/uv.lock ./projects/website/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/website + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ + +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/website/Dockerfile ./projects/website/Dockerfile +COPY ./projects/website/pyproject.toml ./projects/website/pyproject.toml +COPY ./projects/website/uv.lock ./projects/website/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/website + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/website/.venv/bin:$PATH" + +CMD ["uvicorn", "bot_detector.website.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--reload"] + -WORKDIR /app/projects/website +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# install dependencies via RUN uv build -RUN uv sync --frozen --no-editable +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# SAME PATH as builder +WORKDIR /project/projects/website -# Creates a non-root user with an explicit UID and adds permission to access the /project folder -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/website/.venv \ + /project/projects/website/.venv -WORKDIR /app/projects/website -COPY --from=builder --chown=appuser /app/projects/website/.venv /app/projects/website/.venv +ENV PATH="/project/projects/website/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.website.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD ["uvicorn", "bot_detector.website.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/worker_hiscore/Dockerfile b/projects/worker_hiscore/Dockerfile index 4ccbddd..ab68056 100644 --- a/projects/worker_hiscore/Dockerfile +++ b/projects/worker_hiscore/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/worker_hiscore/Dockerfile ./projects/worker_hiscore/Dockerfile +COPY ./projects/worker_hiscore/pyproject.toml ./projects/worker_hiscore/pyproject.toml +COPY ./projects/worker_hiscore/uv.lock ./projects/worker_hiscore/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/worker_hiscore + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/worker_hiscore/Dockerfile ./projects/worker_hiscore/Dockerfile +COPY ./projects/worker_hiscore/pyproject.toml ./projects/worker_hiscore/pyproject.toml +COPY ./projects/worker_hiscore/uv.lock ./projects/worker_hiscore/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/worker_hiscore + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/worker_hiscore/.venv/bin:$PATH" + +CMD ["worker_hiscore"] -WORKDIR /app/projects/worker_hiscore -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/worker_hiscore -WORKDIR /app/projects/worker_hiscore +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/worker_hiscore/.venv \ + /project/projects/worker_hiscore/.venv -COPY --from=builder --chown=appuser /app/projects/worker_hiscore/.venv /app/projects/worker_hiscore/.venv +ENV PATH="/project/projects/worker_hiscore/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/worker_hiscore"] +CMD ["worker_hiscore"] diff --git a/projects/worker_ml/Dockerfile b/projects/worker_ml/Dockerfile index 648d1ed..bcfeebe 100644 --- a/projects/worker_ml/Dockerfile +++ b/projects/worker_ml/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/worker_ml/Dockerfile ./projects/worker_ml/Dockerfile +COPY ./projects/worker_ml/pyproject.toml ./projects/worker_ml/pyproject.toml +COPY ./projects/worker_ml/uv.lock ./projects/worker_ml/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/worker_ml + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/worker_ml/Dockerfile ./projects/worker_ml/Dockerfile +COPY ./projects/worker_ml/pyproject.toml ./projects/worker_ml/pyproject.toml +COPY ./projects/worker_ml/uv.lock ./projects/worker_ml/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/worker_ml + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/worker_ml/.venv/bin:$PATH" + +CMD ["worker_ml"] -WORKDIR /app/projects/worker_ml -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/worker_ml -WORKDIR /app/projects/worker_ml +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/worker_ml/.venv \ + /project/projects/worker_ml/.venv -COPY --from=builder --chown=appuser /app/projects/worker_ml/.venv /app/projects/worker_ml/.venv +ENV PATH="/project/projects/worker_ml/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/worker_ml"] +CMD ["worker_ml"] diff --git a/projects/worker_report/Dockerfile b/projects/worker_report/Dockerfile index 372cd31..87bab27 100644 --- a/projects/worker_report/Dockerfile +++ b/projects/worker_report/Dockerfile @@ -1,30 +1,82 @@ -FROM python:3.12-slim-bookworm AS builder +# ------------------------------------------------------------ +# Base (shared env only, no filesystem assumptions) +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS base + +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /project + + +# ------------------------------------------------------------ +# Builder: production dependencies only +# ------------------------------------------------------------ +FROM base AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv +COPY ./bases ./bases +COPY ./components ./components +COPY ./projects/worker_report/Dockerfile ./projects/worker_report/Dockerfile +COPY ./projects/worker_report/pyproject.toml ./projects/worker_report/pyproject.toml +COPY ./projects/worker_report/uv.lock ./projects/worker_report/uv.lock +COPY workspace.toml workspace.toml + +# IMPORTANT: venv path is fixed here +WORKDIR /project/projects/worker_report + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + + +# ------------------------------------------------------------ +# Dev: development environment with dev dependencies +# ------------------------------------------------------------ +FROM base AS dev -# Copy uv from external repository COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /uvx /bin/ -# Set the working directory for the build stage -WORKDIR /app +ENV PYTHONPATH="/project" -# Copy only necessary files to run the projects COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects +COPY ./projects/worker_report/Dockerfile ./projects/worker_report/Dockerfile +COPY ./projects/worker_report/pyproject.toml ./projects/worker_report/pyproject.toml +COPY ./projects/worker_report/uv.lock ./projects/worker_report/uv.lock +COPY workspace.toml workspace.toml + +WORKDIR /project/projects/worker_report + +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --dev + +ENV PATH="/project/projects/worker_report/.venv/bin:$PATH" + +CMD ["worker_report"] -WORKDIR /app/projects/worker_report -# RUN uv build -RUN uv sync --frozen --no-editable +# ------------------------------------------------------------ +# Prod: minimal, secure, venv-only runtime +# ------------------------------------------------------------ +FROM python:3.12-slim-bookworm AS prod -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Non-root user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser \ + --uid 1000 --disabled-password \ + --no-create-home appuser -RUN adduser -u 5678 --disabled-password --gecos "" appuser +# SAME PATH as builder +WORKDIR /project/projects/worker_report -WORKDIR /app/projects/worker_report +# Copy sealed venv (no relocation!) +COPY --from=builder \ + --chown=appuser:appuser \ + /project/projects/worker_report/.venv \ + /project/projects/worker_report/.venv -COPY --from=builder --chown=appuser /app/projects/worker_report/.venv /app/projects/worker_report/.venv +ENV PATH="/project/projects/worker_report/.venv/bin:$PATH" USER appuser -CMD [".venv/bin/worker_report"] +CMD ["worker_report"]