Airflow, FastAPI and postgres: host with docker

Question

I try to host different services together using docker. For that, I want to write files docker-compose.yml and Dockerfile. They follow below:

version: '3.8'

services:
  postgres:
    image: postgres:latest
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    ports:
      - "${POSTGRES_PORT}:${POSTGRES_PORT}"
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:latest
    ports:
      - "${REDIS_PORT}:${REDIS_PORT}"

  backend-app:
    build: .
    ports:
      - "8000:8000"
    depends_on:
      - postgres
    environment:
      DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/${POSTGRES_DB}

  airflow-scheduler:
    image: apache/airflow:latest
    command: scheduler
    depends_on:
      - redis
      - backend-app
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=${AIRFLOW__CORE__SQL_ALCHEMY_CONN}
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CORE__CELERY_BROKER_URL=redis://${REDIS_PORT}
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=redis://${REDIS_PORT}
      - AIRFLOW__WEBSERVER__RBAC=${AIRFLOW__WEBSERVER__RBAC}
      - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=${AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL}
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins
      
  airflow-webserver:
    image: apache/airflow:latest
    command: webserver
    depends_on:
      - redis
      - backend-app
    ports:
      - "8080:8080"
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=${AIRFLOW__CORE__SQL_ALCHEMY_CONN}
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CORE__CELERY_BROKER_URL=redis://${REDIS_PORT}
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=redis://${REDIS_PORT}
      - AIRFLOW__WEBSERVER__RBAC=${AIRFLOW__WEBSERVER__RBAC}
      - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=${AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL}
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins

networks:
  mynetwork:
    driver: bridge

volumes:
  postgres_data:

# Use the official Python image
FROM python:3.9 

# Set the working directory in the container
WORKDIR /app

# Copy the dependencies file to the working directory
COPY requirements_dev.txt .

# Install any dependencies
RUN pip install -r requirements_dev.txt

# Copy the rest of the application code
COPY . .

# Set up permissions for the log directory
RUN chown -R /opt/airflow/logs/scheduler
RUN mkdir -p /opt/airflow/logs/scheduler \
    && chown -R airflow:airflow /opt/airflow/logs

# Command to run the API
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

After using these files above, the command run docker-compose up on the same folder as desired was succesful to host the postgres and redis databases as well as backend app, but not the airflow schedulers. Do you know why? I do not.

airflow-webserver-1  | ....................
airflow-webserver-1  | ERROR! Maximum number of retries (20) reached.
airflow-webserver-1  | 
airflow-webserver-1  | Last check result:
airflow-webserver-1  | $ airflow db check
airflow-webserver-1  | /home/airflow/.local/lib/python3.8/site-packages/airflow/configuration.py:812 DeprecationWarning: The sql_alchemy_conn option in [core] has been moved to the sql_alchemy_conn option in [database] - the old setting has been used, but please update your config.
airflow-webserver-1  | /home/airflow/.local/lib/python3.8/site-packages/airflow/configuration.py:738 DeprecationWarning: The sql_alchemy_conn option in [core] has been moved to the sql_alchemy_conn option in [database] - the old setting has been used, but please update your config.
airflow-webserver-1  | /home/airflow/.local/lib/python3.8/site-packages/airflow/settings.py:194 DeprecationWarning: The sql_alchemy_conn option in [core] has been moved to the sql_alchemy_conn option in [database] - the old setting has been used, but please update your config.
airflow-webserver-1  | Unable to load the config, contains a configuration error.
airflow-webserver-1  | Traceback (most recent call last):
airflow-webserver-1  |   File "/usr/local/lib/python3.8/logging/config.py", line 563, in configure
airflow-webserver-1  |     handler = self.configure_handler(handlers[name])
airflow-webserver-1  |   File "/usr/local/lib/python3.8/logging/config.py", line 744, in configure_handler
airflow-webserver-1  |     result = factory(**kwargs)
airflow-webserver-1  |   File "/home/airflow/.local/lib/python3.8/site-packages/airflow/utils/log/file_processor_handler.py", line 50, in __init__
airflow-webserver-1  |     Path(self._get_log_directory()).mkdir(parents=True, exist_ok=True)
airflow-webserver-1  |   File "/usr/local/lib/python3.8/pathlib.py", line 1288, in mkdir
airflow-webserver-1  |     self._accessor.mkdir(self, mode)
airflow-webserver-1  | PermissionError: [Errno 13] Permission denied: '/opt/airflow/logs/scheduler/2024-03-27'
airflow-webserver-1  | 
airflow-webserver-1  | The above exception was the direct cause of the following exception:
airflow-webserver-1  | 
airflow-webserver-1  | Traceback (most recent call last):
airflow-webserver-1  |   File "/home/airflow/.local/bin/airflow", line 5, in <module>
airflow-webserver-1  |     from airflow.__main__ import main
airflow-webserver-1  |   File "/home/airflow/.local/lib/python3.8/site-packages/airflow/__init__.py", line 68, in <module>
airflow-webserver-1  |     settings.initialize()
airflow-webserver-1  |   File "/home/airflow/.local/lib/python3.8/site-packages/airflow/settings.py", line 554, in initialize
airflow-webserver-1  |     LOGGING_CLASS_PATH = configure_logging()
airflow-webserver-1  |   File "/home/airflow/.local/lib/python3.8/site-packages/airflow/logging_config.py", line 74, in configure_logging
airflow-webserver-1  |     raise e
airflow-webserver-1  |   File "/home/airflow/.local/lib/python3.8/site-packages/airflow/logging_config.py", line 69, in configure_logging
airflow-webserver-1  |     dictConfig(logging_config)
airflow-webserver-1  |   File "/usr/local/lib/python3.8/logging/config.py", line 808, in dictConfig
airflow-webserver-1  |     dictConfigClass(config).configure()
airflow-webserver-1  |   File "/usr/local/lib/python3.8/logging/config.py", line 570, in configure
airflow-webserver-1  |     raise ValueError('Unable to configure handler '
airflow-webserver-1  | ValueError: Unable to configure handler 'processor'
airflow-webserver-1  | 

I tried to change the folder ownership, but I was not succesful.