Dockerizing Celery and Django

Here is how I dockerized my Django application along with celery. In this project I defined six services:

1. web is the Django dev server
2. db is the Postgres server
3. redis is the Redis service, which will be used as the Celery message broker and result backend
4. celery_worker is the Celery worker process
5. celery_beat is the Celery beat process for scheduled tasks
6. flower is the Celery dashboard

requirements.txt

django==4.0.7
celery==5.2.7
redis==4.3.4
flower==1.2.0
psycopg2-binary==2.9.2

docker-compose.yml

version: '3.8'

services:
  web:
    build:
      context: .
      dockerfile: ./compose/local/django/Dockerfile
    image: django_celery_example_web
    # '/start' is the shell script used to run the service
    command: /start
    # this volume is used to map the files and folders on the host to the container
    # so if we change code on the host, code in the docker container will also be changed
    volumes:
      - .:/app
    ports:
      - 8010:8000
    # env_file is used to manage the env variables of our project
    env_file:
      - ./.env/.dev-sample
    depends_on:
      - redis
      - db

  db:
    image: postgres:14-alpine
    volumes:
      - postgres_data:/var/lib/postgresql/data/
    environment:
      - POSTGRES_DB=hello_django
      - POSTGRES_USER=hello_django
      - POSTGRES_PASSWORD=hello_django

  redis:
    image: redis:7-alpine

  celery_worker:
    build:
      context: .
      dockerfile: ./compose/local/django/Dockerfile
    image: django_celery_example_celery_worker
    command: /start-celeryworker
    volumes:
      - .:/app
    env_file:
      - ./.env/.dev-sample
    depends_on:
      - redis
      - db

  celery_beat:
    build:
      context: .
      dockerfile: ./compose/local/django/Dockerfile
    image: django_celery_example_celery_beat
    command: /start-celerybeat
    volumes:
      - .:/app
    env_file:
      - ./.env/.dev-sample
    depends_on:
      - redis
      - db

  flower:
    build:
      context: .
      dockerfile: ./compose/local/django/Dockerfile
    image: django_celery_example_celery_flower
    command: /start-flower
    volumes:
      - .:/app
    env_file:
      - ./.env/.dev-sample
    ports:
      - 5557:5555
    depends_on:
      - redis
      - db

volumes:
  postgres_data:

settings.py

CELERY_BROKER_URL = os.environ.get("CELERY_BROKER", "redis://127.0.0.1:6379/0")
CELERY_RESULT_BACKEND = os.environ.get("CELERY_BACKEND", "redis://127.0.0.1:6379/0")

Dockerfile

FROM python:3.10-slim-buster

ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1

RUN apt-get update \
  # dependencies for building Python packages
  && apt-get install -y build-essential \
  # psycopg2 dependencies
  && apt-get install -y libpq-dev \
  # Translations dependencies
  && apt-get install -y gettext \
  # cleaning up unused files
  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
  && rm -rf /var/lib/apt/lists/*

# Requirements are installed here to ensure they will be cached.
COPY ./requirements.txt /requirements.txt
RUN pip install -r /requirements.txt

COPY ./compose/local/django/entrypoint /entrypoint
RUN sed -i 's/\r$//g' /entrypoint
RUN chmod +x /entrypoint

COPY ./compose/local/django/start /start
RUN sed -i 's/\r$//g' /start
RUN chmod +x /start

COPY ./compose/local/django/celery/worker/start /start-celeryworker
RUN sed -i 's/\r$//g' /start-celeryworker
RUN chmod +x /start-celeryworker

COPY ./compose/local/django/celery/beat/start /start-celerybeat
RUN sed -i 's/\r$//g' /start-celerybeat
RUN chmod +x /start-celerybeat

COPY ./compose/local/django/celery/flower/start /start-flower
RUN sed -i 's/\r$//g' /start-flower
RUN chmod +x /start-flower

WORKDIR /app

ENTRYPOINT ["/entrypoint"]

Start scripts

compose/local/django/start

#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
python manage.py migrate
python manage.py runserver 0.0.0.0:8000

compose/local/django/celery/beat/start

#!/bin/bash
set -o errexit
set -o nounset
rm -f './celerybeat.pid'
celery -A django_celery_example beat -l INFO

compose/local/django/celery/worker/start

#!/bin/bash
set -o errexit
set -o nounset
celery -A django_celery_example worker -l INFO

compose/local/django/celery/flower/start

#!/bin/bash
set -o errexit
set -o nounset
worker_ready() {
    celery -A django_celery_example inspect ping
}
until worker_ready; do
  >&2 echo 'Celery workers not available'
  sleep 1
done
>&2 echo 'Celery workers is available'
celery -A django_celery_example  \
    --broker="${CELERY_BROKER}" \
    flower

Simple Test

You can test things out by entering the Django shell of the running web service:

$ docker-compose exec web python manage.py shell

Then, run the following code:

>>> from django_celery_example.celery import divide
>>>
>>> divide.delay(1, 2)
<AsyncResult: ea7b49f0-250c-4b21-b3ed-7ad1493e22ee>

View the logs of the Celery worker in a new terminal window

$ docker-compose logs celery_worker

:::::::