diff --git a/.env b/.env new file mode 100644 index 0000000..355db54 --- /dev/null +++ b/.env @@ -0,0 +1,57 @@ +# ============================================================================= +# Environment variables for the django-example application +# Used by docker-compose.yml for all services +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Python / General +# ----------------------------------------------------------------------------- +PYTHONPATH=. +LOGS_LEVEL=DEBUG +ENVIRONMENT=local + +# ----------------------------------------------------------------------------- +# Django Core +# ----------------------------------------------------------------------------- +DJANGO_SECRET_KEY=s8s6s5t2nu00 +DJANGO_DEBUG=True +DJANGO_ADMIN_URL=admin/ +DJANGO_SETTINGS_MODULE=core.settings.local +DJANGO_ALLOWED_HOSTS=* + +# ----------------------------------------------------------------------------- +# PostgreSQL Database +# DB_HOST must match the docker-compose service name: 'db' +# ----------------------------------------------------------------------------- +DB_ENGINE=django.db.backends.postgresql_psycopg2 +DB_NAME=postgres +DB_USER=postgres +DB_PASSWORD=qwerty123 +DB_HOST=db +DB_PORT=5432 + +# PostgreSQL image-level credentials (used by the postgres container itself) +POSTGRES_USER=postgres +POSTGRES_PASSWORD=qwerty123 +POSTGRES_DB=postgres + +# ----------------------------------------------------------------------------- +# Celery + RabbitMQ +# CELERY_BROKER_URL must match the docker-compose service name: 'broker' +# ----------------------------------------------------------------------------- +CELERY_RESULT_BACKEND=django-db +CELERY_BROKER_URL=amqp://admin:admin@broker:5672/vhost +CELERY_WORKER_CONCURRENCY=1 +CELERY_WORKER_PREFETCH_MULTIPLIER=1 +CELERY_WORKER_MAX_TASKS_PER_CHILD=10 +CELERY_RESULT_EXTENDED=False + +# RabbitMQ image-level credentials (used by the rabbitmq container itself) +RABBITMQ_DEFAULT_USER=admin +RABBITMQ_DEFAULT_PASS=admin +RABBITMQ_DEFAULT_VHOST=vhost + +# ----------------------------------------------------------------------------- +# Cache (Redis) — currently disabled, uncomment to enable +# ----------------------------------------------------------------------------- +# REDIS_URL=redis://cache:6379/0 diff --git a/Dockerfile b/Dockerfile index cb77737..5214784 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,115 @@ -FROM python:3.9.13 +# ============================================================================= +# Dockerfile for the Django 'core' service +# Supports both local/development and production environments +# via the ARG ENVIRONMENT build argument. +# +# Usage: +# Local: docker build --build-arg ENVIRONMENT=local -t core:local . +# Production: docker build --build-arg ENVIRONMENT=production -t core:prod . +# ============================================================================= -ARG ENVIRONMENT=default +# ----------------------------------------------------------------------------- +# Base image: Python 3.9.13 (pinned for reproducibility) +# ----------------------------------------------------------------------------- +FROM python:3.9.13 -ENV PYTHONUNBUFFERED 1 \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=1 +# ----------------------------------------------------------------------------- +# Build argument to switch between environments: +# local → installs dev dependencies + telnet, runs Django dev server +# production → installs prod dependencies only, runs Gunicorn +# ----------------------------------------------------------------------------- +ARG ENVIRONMENT=local +# ----------------------------------------------------------------------------- +# Environment variables: +# PYTHONUNBUFFERED → forces stdout/stderr to be unbuffered (better logging) +# PIP_NO_CACHE_DIR → disables pip cache to reduce image size +# PIP_DISABLE_PIP_VERSION_CHECK → suppresses pip upgrade warnings +# ENVIRONMENT → makes the build ARG available at runtime for CMD logic +# ----------------------------------------------------------------------------- +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + ENVIRONMENT=${ENVIRONMENT} +# ----------------------------------------------------------------------------- +# Set the working directory for all subsequent instructions +# ----------------------------------------------------------------------------- WORKDIR /app +# ----------------------------------------------------------------------------- +# Copy only the requirements files first to leverage Docker layer caching. +# Dependencies are re-installed only when requirements files change. +# ----------------------------------------------------------------------------- COPY requirements/*.txt /tmp/requirements/ +# ----------------------------------------------------------------------------- +# Install system and Python dependencies based on the target environment. +# +# buildDeps → required to compile native Python extensions (e.g. psycopg2) +# runDeps → required at runtime (git) +# localDeps → useful for local debugging only (telnet) +# +# In production: build deps are removed after pip install to reduce image size. +# ----------------------------------------------------------------------------- RUN set -x \ - && buildDeps=" \ - build-essential \ - " \ - && runDeps=" \ - git \ - " \ - && localDeps=" \ - telnet \ - " \ - && apt-get update \ - && apt-get install -y --no-install-recommends $buildDeps \ - && apt-get install -y --no-install-recommends $runDeps \ - && if [ $ENVIRONMENT = local ] || [ $ENVIRONMENT = development ]; then \ - apt-get install -y --no-install-recommends $localDeps \ - # Install python dev dependencies - && pip install -r /tmp/requirements/local.txt; \ - else \ - # Install python production dependencies - pip install -r /tmp/requirements/production.txt; \ - # other environment to local remove the build dependencies - apt-get remove -y $buildDeps; \ - fi \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - # clean tmp dir - && rm -rf /tmp/* + && buildDeps="build-essential" \ + && runDeps="git" \ + && localDeps="telnet" \ + # Update apt package index + && apt-get update \ + # Install build-time dependencies (needed to compile psycopg2, etc.) + && apt-get install -y --no-install-recommends $buildDeps \ + # Install runtime dependencies + && apt-get install -y --no-install-recommends $runDeps \ + # Environment-specific Python dependency installation + && if [ "$ENVIRONMENT" = "local" ] || [ "$ENVIRONMENT" = "development" ]; then \ + # Install local debugging tools + apt-get install -y --no-install-recommends $localDeps \ + # Install local/development Python dependencies (includes debug toolbar, etc.) + && pip install -r /tmp/requirements/local.txt; \ + else \ + # Install production Python dependencies only (gunicorn, no debug tools) + pip install -r /tmp/requirements/production.txt \ + # Remove build dependencies to keep the production image lean + && apt-get remove -y $buildDeps \ + && apt-get autoremove -y; \ + fi \ + # Clean up apt cache to reduce image size + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + # Remove temporary requirements files + && rm -rf /tmp/* +# ----------------------------------------------------------------------------- +# Copy the full application source code into the container. +# Done after dependency installation to maximise Docker layer cache reuse. +# ----------------------------------------------------------------------------- COPY . . -CMD [ "python","manage.py","runserver","0.0.0.0:8000" ] +# ----------------------------------------------------------------------------- +# Ensure the entrypoint script is executable. +# The entrypoint handles: +# 1. Waiting for PostgreSQL to be ready (via psycopg2 health check) +# 2. Running makemigrations and migrate +# 3. Loading initial data if this is the first run +# 4. Starting the Django development server +# ----------------------------------------------------------------------------- +RUN chmod +x /app/docker-entrypoint.sh + +# ----------------------------------------------------------------------------- +# Startup command: +# local/development → runs docker-entrypoint.sh (dev server on port 8000) +# production → runs Gunicorn WSGI server directly (multi-worker) +# Note: In production, DB migrations should be handled +# separately (e.g. init container or CI/CD pipeline). +# ----------------------------------------------------------------------------- +CMD if [ "$ENVIRONMENT" = "production" ]; then \ + gunicorn core.wsgi:application \ + --bind 0.0.0.0:8000 \ + --workers 2 \ + --timeout 120 \ + --log-level info; \ + else \ + /app/docker-entrypoint.sh; \ + fi diff --git a/Dockerfile.celeryworker b/Dockerfile.celeryworker new file mode 100644 index 0000000..0ffc3be --- /dev/null +++ b/Dockerfile.celeryworker @@ -0,0 +1,108 @@ +# ============================================================================= +# Dockerfile for the celeryworker service +# Asynchronous task worker using Celery with Django integration +# Supports both local/development and production environments via ARG ENVIRONMENT +# ============================================================================= + +# Base image: Python 3.9.13 (matches project's required Python version) +FROM python:3.9.13 + +# Build argument to switch between local and production configurations +# Values: local | development | production (default: local) +ARG ENVIRONMENT=local + +# ----------------------------------------------------------------------------- +# Environment variables +# PYTHONUNBUFFERED: Ensures Python output is sent straight to terminal (no buffering) +# PIP_NO_CACHE_DIR: Disables pip cache to reduce image size +# PIP_DISABLE_PIP_VERSION_CHECK: Suppresses pip version warnings during builds +# ----------------------------------------------------------------------------- +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Set the working directory inside the container +WORKDIR /app + +# ----------------------------------------------------------------------------- +# Copy only the requirements files first to leverage Docker layer caching. +# This way, dependencies are only re-installed when requirements files change. +# ----------------------------------------------------------------------------- +COPY requirements/*.txt /tmp/requirements/ + +# ----------------------------------------------------------------------------- +# Install system and Python dependencies based on the target environment. +# +# buildDeps: Packages needed only at build time (removed in production to reduce image size) +# runDeps: Packages needed at runtime in all environments +# localDeps: Extra packages useful only in local/development (e.g. telnet for debugging) +# +# For local/development: +# - Installs buildDeps + runDeps + localDeps +# - Installs Python deps from requirements/local.txt (_base.txt + django-debug-toolbar) +# +# For production: +# - Installs buildDeps + runDeps only +# - Installs Python deps from requirements/production.txt (_base.txt + gunicorn) +# - Removes buildDeps after install to keep the image lean +# ----------------------------------------------------------------------------- +RUN set -x \ + && buildDeps=" \ + build-essential \ + " \ + && runDeps=" \ + git \ + " \ + && localDeps=" \ + telnet \ + " \ + # Update apt package index + && apt-get update \ + # Install build-time dependencies + && apt-get install -y --no-install-recommends $buildDeps \ + # Install runtime dependencies + && apt-get install -y --no-install-recommends $runDeps \ + # Conditionally install environment-specific packages and Python deps + && if [ "$ENVIRONMENT" = "local" ] || [ "$ENVIRONMENT" = "development" ]; then \ + # Install local debugging tools + apt-get install -y --no-install-recommends $localDeps \ + # Install Python dependencies for local environment (includes django-debug-toolbar) + && pip install -r /tmp/requirements/local.txt; \ + else \ + # Install Python dependencies for production (includes gunicorn) + pip install -r /tmp/requirements/production.txt \ + # Remove build dependencies to reduce final image size in production + && apt-get remove -y $buildDeps; \ + fi \ + # Clean up apt cache to reduce image size + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + # Remove temporary requirements files + && rm -rf /tmp/* + +# ----------------------------------------------------------------------------- +# Copy the full application source code into the container. +# Done after dependency installation to maximize Docker layer cache reuse. +# ----------------------------------------------------------------------------- +COPY . . + +# ----------------------------------------------------------------------------- +# Default startup command for the Celery worker. +# +# Flags: +# -A core.celery : Django project's Celery application module +# worker : Start a Celery worker process +# -l INFO : Set log level to INFO +# --concurrency 1 : Use 1 worker process (suitable for CPU-bound tasks) +# --max-tasks-per-child 1 : Restart worker after each task (prevents memory leaks) +# --prefetch-multiplier 1 : Fetch only 1 task at a time per worker (fair dispatch) +# -n celery@%h : Worker node name using the hostname +# +# Override this CMD in docker-compose or at runtime for production tuning. +# ----------------------------------------------------------------------------- +CMD ["celery", "-A", "core.celery", "worker", \ + "-l", "INFO", \ + "--concurrency", "1", \ + "--max-tasks-per-child", "1", \ + "--prefetch-multiplier", "1", \ + "-n", "celery@%h"] diff --git a/Dockerfile.flower b/Dockerfile.flower new file mode 100644 index 0000000..c04fb23 --- /dev/null +++ b/Dockerfile.flower @@ -0,0 +1,92 @@ +# ============================================================================= +# Dockerfile for the Flower service (Celery task monitoring) +# Flower is a web-based tool for monitoring and administrating Celery clusters. +# Supports two environments via ARG ENVIRONMENT: local | production +# ============================================================================= + +# Base image: Python 3.9.13 (matches the project's required runtime) +FROM python:3.9.13 + +# Build argument to switch between local and production environments +# Usage: docker build --build-arg ENVIRONMENT=production ... +ARG ENVIRONMENT=local + +# ----------------------------------------------------------------------------- +# Environment variables +# - PYTHONUNBUFFERED: Ensures Python output is sent straight to the terminal +# without buffering, which is important for logging in containers. +# - PIP_NO_CACHE_DIR: Disables pip cache to reduce image size. +# - PIP_DISABLE_PIP_VERSION_CHECK: Suppresses pip version warnings. +# ----------------------------------------------------------------------------- +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Set the working directory inside the container +WORKDIR /app + +# ----------------------------------------------------------------------------- +# Copy only the requirements files first to leverage Docker layer caching. +# This way, dependencies are only re-installed when requirements files change. +# ----------------------------------------------------------------------------- +COPY requirements/*.txt /tmp/requirements/ + +# ----------------------------------------------------------------------------- +# Install system dependencies and Python packages. +# - build-essential: Required to compile native Python extensions (e.g. psycopg2) +# - git: Runtime dependency for some Python packages +# - telnet: Useful for debugging connectivity in local environments only +# The build dependencies (build-essential) are removed in production to keep +# the image lean. Local environments retain them for development convenience. +# ----------------------------------------------------------------------------- +RUN set -x \ + && buildDeps="build-essential" \ + && runDeps="git" \ + && localDeps="telnet" \ + && apt-get update \ + # Install build dependencies (needed to compile psycopg2, etc.) + && apt-get install -y --no-install-recommends $buildDeps \ + # Install runtime dependencies + && apt-get install -y --no-install-recommends $runDeps \ + # Install environment-specific dependencies and Python packages + && if [ "$ENVIRONMENT" = "local" ] || [ "$ENVIRONMENT" = "development" ]; then \ + # Install telnet for local debugging + apt-get install -y --no-install-recommends $localDeps \ + # Install local Python dependencies (includes debug toolbar, etc.) + && pip install -r /tmp/requirements/local.txt; \ + else \ + # Install production Python dependencies (includes gunicorn) + pip install -r /tmp/requirements/production.txt \ + # Remove build dependencies in production to reduce image size + && apt-get remove -y $buildDeps; \ + fi \ + # Clean up APT cache to reduce image size + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + # Clean up temporary requirements files + && rm -rf /tmp/* + +# ----------------------------------------------------------------------------- +# Copy the full application source code into the container. +# This includes the Django project, Celery configuration, and all app modules +# required by Flower to introspect tasks and connect to the broker. +# ----------------------------------------------------------------------------- +COPY . . + +# ----------------------------------------------------------------------------- +# Expose the Flower web UI port. +# Flower runs on port 5555 by default. +# ----------------------------------------------------------------------------- +EXPOSE 5555 + +# ----------------------------------------------------------------------------- +# Default startup command for Flower. +# Starts the Flower monitoring server connected to the Celery app defined +# in core/celery.py. The broker URL and other settings are provided via +# environment variables at runtime (CELERY_BROKER_URL, DJANGO_SETTINGS_MODULE). +# +# Local: Uses the same command (Flower has no separate prod/local mode) +# Production: Same command; broker URL and credentials differ via env vars +# ----------------------------------------------------------------------------- +CMD ["celery", "-A", "core.celery", "flower", "--port=5555"] diff --git a/README.md b/README.md index 723c903..49a9c59 100644 --- a/README.md +++ b/README.md @@ -1,535 +1,267 @@ -# example-django-celery +# Django Example Project -Complete Django + Celery example with RabbitMQ broker for asynchronous task processing. +A complete Django web application with Celery task processing, PostgreSQL database, RabbitMQ message broker, and Flower monitoring. -## Getting Started +## 🚀 Quick Start ### Prerequisites -- Docker and Docker Compose installed -- Make utility - -### Running the Project - -1. Copy environment file: - ```bash - cp .env.local .env - ``` - -2. Start all services (Django, PostgreSQL, RabbitMQ, Celery Worker, Flower): - ```bash - make run - ``` - This will build images on first run and start: - - **Django app**: `http://localhost:8000` - - **PostgreSQL**: Database on port `5432` - - **RabbitMQ**: Broker on port `5672` (Management UI: `http://localhost:15672`) - - **Celery Worker**: Background task processor - - **Flower**: Task monitoring dashboard at `http://localhost:5555` - -3. Create superuser to access Django admin: - ```bash - make createsuperuser - ``` - -4. Access the application: - - **Django Admin**: http://localhost:8000/admin/ - - **Flower Dashboard**: http://localhost:5555 - -### Available Make Commands - -Run `make help` to see all available commands: -- `make run` - Start all services -- `make stop` - Stop all containers -- `make bash` - Access bash inside core container -- `make shell` - Access Django shell -- `make migrate` - Run database migrations -- `make makemigrations` - Create new migrations -- `make reset-db` - Reset database (⚠️ deletes all data) -- `make rebuild-core` - Rebuild core container from scratch -- `make rebuild-all` - Rebuild everything - -## Project Structure -``` -├── apps/ -│ ├── client/ # Client app -│ │ ├── tasks.py # Celery tasks definitions -│ │ ├── admin.py # Admin actions that trigger tasks -│ │ └── views.py # Views (can trigger tasks) -│ ├── post/ # Post app -│ └── user/ # User app -├── core/ -│ ├── celery.py # Celery configuration -│ └── settings/ # Django settings -├── docker-compose.yml # All services orchestration -├── Makefile # Helper commands -└── .env.local # Environment variables template -``` - ---- - -## How to Install & Configure Celery (Step by Step) +- Docker Engine 20.10+ +- Docker Compose 2.0+ +- Git -If you want to add Celery to your own Django project, follow these steps: +### 1. Clone and Setup -### Step 1: Install Dependencies - -Add to your `requirements/_base.txt` or `requirements.txt`: - -```txt -celery==5.3.6 # Core Celery library -django-celery-results==2.5.1 # Optional: Store results in DB -flower==2.0.1 # Optional: Monitoring dashboard -``` - -**Install:** ```bash -pip install -r requirements.txt -``` - -### Step 2: Configure Django Settings - -Add Celery configuration to `core/settings/base.py`: - -```python -# Celery Configuration -# https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html - -CELERY_BROKER_URL = env.str("CELERY_BROKER_URL") # RabbitMQ or Redis URL -CELERY_RESULT_BACKEND = env.str("CELERY_RESULT_BACKEND") # Optional: "django-db" to store results -CELERY_WORKER_PREFETCH_MULTIPLIER = env.int("CELERY_WORKER_PREFETCH_MULTIPLIER", default=1) -CELERY_RESULT_EXTENDED = env.bool("CELERY_RESULT_EXTENDED", default=False) # Optional: Extended result info -``` - -**Important notes:** -- `CELERY_RESULT_BACKEND` and `CELERY_RESULT_EXTENDED` are **optional** - only needed if you want to store task results in the database -- If using `django-celery-results`, add `"django_celery_results"` to `INSTALLED_APPS` - -Reference: [`core/settings/base.py:138-143`](https://github.com/sleakops/example-django-celery/blob/main/core/settings/base.py#L138) - -### Step 3: Create Celery App - -Create `core/celery.py`: - -```python -import os -from celery import Celery -from django.conf import settings - -# Set default Django settings module -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings.local") - -# Create Celery app -app = Celery("core") - -# Load config from Django settings with CELERY_ namespace -app.config_from_object(settings, namespace="CELERY") - -# Auto-discover tasks.py in all Django apps -app.autodiscover_tasks() +git clone +cd django-example ``` -Reference: [`core/celery.py`](https://github.com/sleakops/example-django-celery/blob/main/core/celery.py) +### 2. Environment Configuration -### Step 4: Initialize Celery on Django Startup +Create your `.env` file from the provided template: -Modify `core/__init__.py` to import Celery app: - -```python -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from .celery import app as celery_app - -__all__ = ('celery_app',) +```bash +cp .env.local .env ``` -### Step 5: Define Tasks +The `.env` file contains all necessary configuration. Default values are pre-configured for local development. -Create `tasks.py` in any Django app (e.g., `apps/client/tasks.py`): +### 3. Build and Start Services -```python -from celery import shared_task - -@shared_task -def my_async_task(param): - # Your task logic here - print(f"Processing: {param}") - return "Task completed" +```bash +docker compose up -d ``` -**Magic:** Celery automatically discovers any `tasks.py` file in your Django apps! +This will start all services: +- **PostgreSQL** database (port 5432) +- **RabbitMQ** message broker (ports 5672, 15672) +- **Django** web application (port 8000) +- **Celery** worker +- **Flower** monitoring UI (port 5555) -Reference: [`apps/client/tasks.py`](https://github.com/sleakops/example-django-celery/blob/main/apps/client/tasks.py) - -### Step 6: Set Environment Variables - -Add to your `.env` file: +### 4. Verify Services +Check service status: ```bash -# Celery Configuration -CELERY_BROKER_URL=amqp://admin:admin@broker:5672/vhost # RabbitMQ -# OR -# CELERY_BROKER_URL=redis://localhost:6379/0 # Redis - -CELERY_RESULT_BACKEND=django-db # Optional: Store results in database +docker compose ps ``` -### Step 7: Run Celery Worker - +View logs: ```bash -# Development -celery -A core.celery worker -l INFO - -# Production (with more options) -celery -A core.celery worker -l INFO --concurrency 4 --max-tasks-per-child 100 +docker compose logs -f ``` -**In Docker (see `docker-compose.yml:53-70`):** -```yaml -celeryworker: - image: django-example/core - command: celery -A core.celery worker -l INFO --concurrency 1 - depends_on: - - db - - broker -``` - -### Step 8: Trigger Tasks +### 5. Access the Application -From anywhere in your Django code: +| Service | URL | Description | +|---------|-----|-------------| +| Django App | http://localhost:8000 | Main web application | +| Django Admin | http://localhost:8000/admin/ | Admin panel | +| Flower UI | http://localhost:5555 | Celery task monitoring | +| RabbitMQ UI | http://localhost:15672 | Message broker management | -```python -from apps.client.tasks import my_async_task +**Default RabbitMQ credentials:** +- Username: `admin` +- Password: `admin` -# Asynchronous execution (non-blocking) -task = my_async_task.delay("parameter") -print(f"Task ID: {task.id}") +## 📁 Project Structure -# With custom options -task = my_async_task.apply_async( - args=["parameter"], - countdown=10, # Execute after 10 seconds -) ``` - ---- - -## Celery Architecture - -This project demonstrates the complete Celery workflow for asynchronous task processing: - -``` -Django View/Admin → Celery Task (.delay) → RabbitMQ Broker → Celery Worker → Result Backend (DB) +. +├── apps/ # Django applications +│ ├── base/ # Base app with shared utilities +│ ├── client/ # Client management app +│ ├── post/ # Post/content app +│ └── user/ # User management app +├── core/ # Django project configuration +│ ├── settings/ # Environment-specific settings +│ │ ├── base.py # Base configuration +│ │ ├── local.py # Local development settings +│ │ └── production.py # Production settings +│ ├── celery.py # Celery configuration +│ ├── urls.py # URL routing +│ ├── wsgi.py # WSGI entry point +│ └── asgi.py # ASGI entry point +├── requirements/ # Python dependencies +│ ├── _base.txt # Base dependencies +│ ├── local.txt # Local development extras +│ ├── production.txt # Production extras +│ └── test.txt # Test dependencies +├── Dockerfile # Main Django service image +├── Dockerfile.celeryworker # Celery worker image +├── Dockerfile.flower # Flower monitoring image +├── docker-compose.yml # Service orchestration +├── docker-entrypoint.sh # Container startup script +├── .env.local # Environment template +└── README.md # This file ``` -### Components +## 🔧 Development -#### 1. **Broker (RabbitMQ)** -Message broker that queues tasks between Django and workers. +### Running Management Commands -- **Configuration** (`.env.local`): - ```bash - CELERY_BROKER_URL=amqp://admin:admin@broker:5672/vhost - ``` -- **Docker service** (`docker-compose.yml:20-30`): RabbitMQ with management plugin -- **Ports**: - - `5672` - AMQP protocol - - `15672` - Management UI (http://localhost:15672, user: `admin`, pass: `admin`) +```bash +# Create superuser +docker compose exec core python manage.py createsuperuser -#### 2. **Celery App** (`core/celery.py`) -Main Celery application configuration. +# Run migrations manually +docker compose exec core python manage.py migrate -```python -app = Celery("core") -app.config_from_object(settings, namespace="CELERY") -app.autodiscover_tasks() # Finds tasks.py in all Django apps -``` +# Django shell +docker compose exec core python manage.py shell -- Auto-discovers tasks in `tasks.py` from all registered Django apps -- Loaded on Django startup via `core/__init__.py:3` -- Namespace `CELERY_` means all config uses this prefix in `.env` - -#### 3. **Tasks** (`apps/client/tasks.py`) -Asynchronous task definitions using `@shared_task` decorator. - -**Example task** (`apps/client/tasks.py:7-17`): -```python -@shared_task -def collect_post_by_client(client_id): - client = Client.objects.get(id=client_id) - print(f'Request: {client.username}') - - for i in range(1, 200): - Post.objects.create( - client=client, - source="twitter", - description=f"Lorem {i}" - ) +# Check logs for a specific service +docker compose logs -f core ``` -This task creates 200 posts asynchronously, preventing request timeout. +### Database Migrations -#### 4. **Result Backend** -Stores task results and status in database. +Migrations run automatically on container startup. To create new migrations: -- **Configuration** (`.env.local:23`): - ```bash - CELERY_RESULT_BACKEND="django-db" - ``` -- **Package**: `django-celery-results` (installed in `requirements.txt:6`) -- **Purpose**: Track task status (PENDING, STARTED, SUCCESS, FAILURE) and retrieve results +```bash +docker compose exec core python manage.py makemigrations +``` -#### 5. **Celery Worker** (`docker-compose.yml:53-70`) -Background process that executes tasks from the queue. +### Static Files -```yaml -celeryworker: - command: celery -A core.celery worker -l INFO --concurrency 1 -``` +In local development, static files are served automatically. For production: -- Runs in separate Docker container -- Shares same codebase as Django app -- Configuration: - - `--concurrency 1`: Process 1 task at a time - - `--max-tasks-per-child 1`: Restart worker after each task (prevents memory leaks) - - `--prefetch-multiplier 1`: Don't prefetch tasks - -#### 6. **Flower** (`docker-compose.yml:72-87`) -Real-time monitoring dashboard for Celery. - -```yaml -flower: - command: celery -A core.celery flower - ports: - - "5555:5555" +```bash +docker compose exec core python manage.py collectstatic ``` -- **Access**: http://localhost:5555 -- **Features**: Monitor tasks, workers, queue status, task history, and execution times +## 🧪 Testing ---- +```bash +# Run Django tests +docker compose exec core python manage.py test -## How to Trigger Tasks +# Run tests with coverage +docker compose exec core python manage.py test --verbosity=2 +``` -### Option 1: From Django Admin (Already Implemented) +## 🛑 Stopping Services -The project includes a Django admin action in `apps/client/admin.py:10-19`: +```bash +# Stop all services +docker compose down -```python -@admin.action(description="Run collect post task") -def collect_post(modeladmin, request, queryset): - for c in queryset.all(): - collect_post_by_client.delay(c.id) # Sends task to broker - messages.add_message( - request, - messages.INFO, - f"Run collect post task for client {c.username}", - ) +# Stop and remove volumes (WARNING: deletes database data) +docker compose down -v ``` -**Usage Steps:** -1. Go to http://localhost:8000/admin/client/client/ -2. Select one or more clients from the list -3. Choose **"Run collect post task"** from the actions dropdown -4. Click "Go" -5. The task is sent to RabbitMQ and processed by Celery worker -6. Monitor progress in Flower: http://localhost:5555 - -### Option 2: From a Django View (Code Example) - -You can trigger tasks from any Django view. Here's a complete example: - -```python -# apps/client/views.py -from django.http import JsonResponse -from celery.result import AsyncResult -from .tasks import collect_post_by_client - -def trigger_collect_posts(request, client_id): - """Trigger async task to collect posts for a client""" - task = collect_post_by_client.delay(client_id) - return JsonResponse({ - 'task_id': task.id, - 'status': 'Task sent to broker', - 'client_id': client_id, - 'monitor_url': f'http://localhost:5555/task/{task.id}' - }) - -def check_task_status(request, task_id): - """Check the status of a running task""" - task = AsyncResult(task_id) - - response_data = { - 'task_id': task_id, - 'status': task.status, # PENDING, STARTED, SUCCESS, FAILURE, RETRY - 'ready': task.ready(), - } - - if task.ready(): - if task.successful(): - response_data['result'] = task.result - else: - response_data['error'] = str(task.info) - - return JsonResponse(response_data) -``` +## 🔐 Environment Variables -**Add to `core/urls.py`:** +Key environment variables (defined in `.env`): -```python -from django.urls import path -from apps.client import views +| Variable | Description | Default | +|----------|-------------|---------| +| `DJANGO_SECRET_KEY` | Django security key | Auto-generated | +| `DJANGO_DEBUG` | Debug mode | `True` | +| `DJANGO_SETTINGS_MODULE` | Settings module path | `core.settings.local` | +| `DB_NAME` | PostgreSQL database name | `postgres` | +| `DB_USER` | PostgreSQL username | `postgres` | +| `DB_PASSWORD` | PostgreSQL password | `qwerty123` | +| `DB_HOST` | PostgreSQL host | `db` | +| `DB_PORT` | PostgreSQL port | `5432` | +| `CELERY_BROKER_URL` | RabbitMQ connection URL | `amqp://admin:admin@broker:5672/vhost` | +| `CELERY_RESULT_BACKEND` | Celery result storage | `django-db` | -urlpatterns = [ - # ... existing patterns - path('api/collect-posts//', views.trigger_collect_posts, name='trigger_collect_posts'), - path('api/task-status//', views.check_task_status, name='check_task_status'), -] -``` +See `.env.local` for the complete list. -**Usage:** -```bash -# Trigger task -curl http://localhost:8000/api/collect-posts/1/ - -# Response: -# { -# "task_id": "a7f3e0b2-5c8d-4e9f-b1a2-3d4e5f6a7b8c", -# "status": "Task sent to broker", -# "client_id": 1, -# "monitor_url": "http://localhost:5555/task/a7f3e0b2-5c8d-4e9f-b1a2-3d4e5f6a7b8c" -# } - -# Check task status -curl http://localhost:8000/api/task-status/a7f3e0b2-5c8d-4e9f-b1a2-3d4e5f6a7b8c/ - -# Response: -# { -# "task_id": "a7f3e0b2-5c8d-4e9f-b1a2-3d4e5f6a7b8c", -# "status": "SUCCESS", -# "ready": true -# } -``` +## 🏗️ Production Deployment -### Option 3: From Django Shell +### Build Production Images ```bash -make shell +docker compose build --build-arg ENVIRONMENT=production ``` -```python -from apps.client.tasks import collect_post_by_client +### Production Configuration -# Trigger task -task = collect_post_by_client.delay(1) -print(f"Task ID: {task.id}") +1. Update `.env` with production values: + - Set `ENVIRONMENT=production` + - Set `DJANGO_DEBUG=False` + - Generate a secure `DJANGO_SECRET_KEY` + - Configure `DJANGO_ALLOWED_HOSTS` + - Set up AWS credentials for S3 storage -# Check status -print(f"Status: {task.status}") +2. Use a production-grade database (AWS RDS, etc.) -# Wait for result (blocking) -result = task.get(timeout=60) -``` +3. Configure SSL/TLS certificates ---- - -## Complete Task Flow Example +4. Set up a reverse proxy (nginx, traefik) -Here's what happens when you trigger a task: +### AWS Managed Services -1. **User Action**: Admin selects a client and clicks "Run collect post task" -2. **Django**: Calls `collect_post_by_client.delay(client_id)` (non-blocking) -3. **Celery**: Serializes task (function name + arguments) and sends to RabbitMQ -4. **RabbitMQ**: Stores task message in queue -5. **Celery Worker**: Picks up task from queue and executes `collect_post_by_client()` -6. **Task Execution**: Creates 200 posts in PostgreSQL database -7. **Result Storage**: Saves task result/status in `django_celery_results` table -8. **Monitoring**: View real-time progress in Flower dashboard +This application can leverage AWS managed services: -**Timing**: -- Django response: ~50ms (immediately returns after queuing) -- Task execution: ~5-10 seconds (running in background) -- Database operations: 200 inserts +| Component | AWS Service | +|-----------|-------------| +| PostgreSQL | Amazon RDS for PostgreSQL | +| RabbitMQ | Amazon MQ (RabbitMQ engine) or Amazon SQS | +| Static/Media Files | Amazon S3 | +| Email | Amazon SES | +| Cache | Amazon ElastiCache for Redis | ---- +## 🐛 Troubleshooting -## Environment Configuration - -Key Celery settings in `.env.local`: +### Services Won't Start ```bash -# Celery Configuration -CELERY_RESULT_BACKEND="django-db" # Store results in PostgreSQL -CELERY_BROKER_URL=amqp://admin:admin@broker:5672/vhost # RabbitMQ connection -CELERY_WORKER_CONCURRENCY=1 # Number of worker processes -CELERY_WORKER_PREFETCH_MULTIPLIER=1 # Tasks to prefetch per worker -CELERY_WORKER_MAX_TASKS_PER_CHILD=10 # Restart worker after N tasks -``` +# Check for port conflicts +docker ps ---- - -## Monitoring & Debugging +# View detailed logs +docker compose logs -### Flower Dashboard -Access http://localhost:5555 to: -- View active workers -- Monitor task queue -- Check task execution history -- See task success/failure rates -- Inspect individual task details +# Restart a specific service +docker compose restart +``` -### RabbitMQ Management UI -Access http://localhost:15672 (user: `admin`, pass: `admin`) to: -- View message queues -- Check connection status -- Monitor message rates -- Inspect queue bindings +### Database Connection Issues -### Check Celery Worker Logs ```bash -docker logs django-example-celeryworker -f -``` +# Check if database is healthy +docker compose ps db -### Check Django Logs -```bash -docker logs django-example-core -f +# View database logs +docker compose logs db ``` ---- - -## Troubleshooting +### Celery Tasks Not Processing -**Workers not processing tasks?** ```bash -# Check worker status -docker ps | grep celeryworker +# Check Celery worker logs +docker compose logs celeryworker -# Restart worker -docker restart django-example-celeryworker +# Check Flower UI for task status +# http://localhost:5555 + +# Restart Celery worker +docker compose restart celeryworker ``` -**RabbitMQ connection issues?** -```bash -# Check broker is running -docker ps | grep broker +## 📚 Additional Resources -# Check environment variable -docker exec django-example-core env | grep CELERY_BROKER_URL -``` +- [Django Documentation](https://docs.djangoproject.com/) +- [Celery Documentation](https://docs.celeryq.dev/) +- [Flower Documentation](https://flower.readthedocs.io/) +- [RabbitMQ Documentation](https://www.rabbitmq.com/documentation.html) +- [Docker Compose Documentation](https://docs.docker.com/compose/) -**Tasks stuck in PENDING?** -- Verify worker is running: `docker ps | grep celeryworker` -- Check worker logs: `docker logs django-example-celeryworker` -- Ensure broker URL is correct in `.env` +## 📝 License -**Database errors in tasks?** -- Check PostgreSQL is running: `docker ps | grep db` -- Verify database migrations: `make migrate` +This project is for educational purposes. ---- +## 🤝 Contributing -## Health Check +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +--- -- **Port**: 8000 -- **Path**: `/healthcheck/` (if implemented) -- **Expected Status**: 200 +**Happy Coding! 🎉** diff --git a/docker-compose.yml b/docker-compose.yml index 8822484..62574d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,92 +1,193 @@ -version: "3.3" +# ============================================================================= +# docker-compose.yml for django-example +# +# Services: +# db → PostgreSQL 14.1 (relational database) +# broker → RabbitMQ 3 with management UI (Celery message broker) +# core → Django web application (port 8000) +# celeryworker → Celery asynchronous task worker +# flower → Celery monitoring web UI (port 5555) +# +# All environment variables are loaded from the .env file. +# Run with: docker compose up -d +# ============================================================================= + +version: "3.9" + services: + + # --------------------------------------------------------------------------- + # PostgreSQL 14.1 — Primary relational database + # Data is persisted in the named volume 'db_data' + # --------------------------------------------------------------------------- db: image: postgres:14.1 - env_file: .env container_name: django-example-db - environment: - - POSTGRES_USER=${DB_USER} - - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_DB=${DB_NAME} + restart: unless-stopped + # Load all env vars from .env; postgres image reads POSTGRES_* vars + env_file: + - .env ports: - - ${DB_PORT}:${DB_PORT} + # Expose PostgreSQL on host port 5432 for local DB tools (e.g. pgAdmin) + - "5432:5432" volumes: - - db:/var/lib/postgresql/data - - # cache: - # image: redis:7.0.0-alpine - # restart: always + # Persist database data across container restarts + - db_data:/var/lib/postgresql/data + healthcheck: + # pg_isready checks if PostgreSQL is accepting connections + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + # --------------------------------------------------------------------------- + # RabbitMQ 3 with Management UI — Message broker for Celery + # Management UI available at http://localhost:15672 (admin/admin) + # --------------------------------------------------------------------------- broker: - image: rabbitmq:3-management # AWS MQ, SQS - restart: always + image: rabbitmq:3-management container_name: django-example-broker - environment: - - RABBITMQ_DEFAULT_USER=admin - - RABBITMQ_DEFAULT_PASS=admin - - RABBITMQ_DEFAULT_VHOST=vhost + restart: unless-stopped + # Load all env vars from .env; rabbitmq image reads RABBITMQ_DEFAULT_* vars + env_file: + - .env ports: - - 5672:5672 - - 15672:15672 + # AMQP protocol port — used by Celery workers and the Django app + - "5672:5672" + # RabbitMQ Management UI port + - "15672:15672" + healthcheck: + # rabbitmq-diagnostics ping verifies the broker node is running + test: ["CMD", "rabbitmq-diagnostics", "ping"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 40s + # --------------------------------------------------------------------------- + # Django core web application + # Built from Dockerfile (local environment) + # Starts via docker-entrypoint.sh which: waits for DB, runs migrations, + # loads initial data, then starts the Django development server on port 8000 + # --------------------------------------------------------------------------- core: build: context: . dockerfile: Dockerfile args: + # Use 'local' for dev server; change to 'production' for Gunicorn ENVIRONMENT: local - image: django-example/core - command: ./docker-entrypoint.sh + image: core:local container_name: django-example-core - env_file: .env - stdin_open: true - tty: true - depends_on: - - db - - broker - # - cache - volumes: - - .:/app + restart: unless-stopped + env_file: + - .env ports: + # Django dev server accessible at http://localhost:8000 - "8000:8000" + depends_on: + # Wait for db and broker to be healthy before starting core + db: + condition: service_healthy + broker: + condition: service_healthy + stdin_open: true + tty: true + healthcheck: + # Use the explicit /healthcheck/ endpoint defined in core/urls.py + test: ["CMD", "curl", "-f", "http://localhost:8000/healthcheck/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + # --------------------------------------------------------------------------- + # Celery worker — Asynchronous background task processor + # Built from Dockerfile.celeryworker (local environment) + # Connects to RabbitMQ broker and stores results in Django DB + # --------------------------------------------------------------------------- celeryworker: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.celeryworker args: + # Use 'local' for development; change to 'production' for prod tuning ENVIRONMENT: local - image: django-example/core + image: celeryworker:local container_name: django-example-celeryworker - platform: linux/x86_64 - volumes: - - .:/app:z + restart: unless-stopped env_file: - .env depends_on: - - db - - broker - # - cache - command: celery -A core.celery worker -l INFO --concurrency 1 --max-tasks-per-child 1 --prefetch-multiplier 1 -n celery@%h + # Wait for db and broker to be healthy before starting the worker + db: + condition: service_healthy + broker: + condition: service_healthy + command: + # Start Celery worker with controlled concurrency settings + - celery + - -A + - core.celery + - worker + - -l + - INFO + - --concurrency=1 + - --max-tasks-per-child=1 + - --prefetch-multiplier=1 + - -n + - celery@%h + healthcheck: + # Celery inspect ping verifies the worker is alive and connected to broker + test: ["CMD-SHELL", "celery -A core.celery inspect ping -d celery@$$HOSTNAME || exit 1"] + interval: 30s + timeout: 15s + retries: 3 + start_period: 60s + # --------------------------------------------------------------------------- + # Flower — Web-based Celery monitoring and administration UI + # Built from Dockerfile.flower (local environment) + # Accessible at http://localhost:5555 + # --------------------------------------------------------------------------- flower: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.flower args: + # Use 'local' for development; change to 'production' for prod ENVIRONMENT: local - image: django-example/core + image: flower:local container_name: django-example-flower - platform: linux/x86_64 - volumes: - - .:/app:z + restart: unless-stopped env_file: - .env ports: + # Flower web UI accessible at http://localhost:5555 - "5555:5555" - command: celery -A core.celery flower - + depends_on: + # Wait for broker to be healthy before starting Flower + broker: + condition: service_healthy + command: + # Start Flower connected to the Celery app and listen on port 5555 + - celery + - -A + - core.celery + - flower + - --port=5555 + healthcheck: + # Flower exposes a /healthcheck endpoint on its web UI port + test: ["CMD", "curl", "-f", "http://localhost:5555/healthcheck"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +# ============================================================================= +# Named volumes for data persistence +# ============================================================================= volumes: - db: {} - # cache: {} + # PostgreSQL data — persists database across container restarts + db_data: {}