Random Bad Gateway

Traefik Traefik v3 (beta)
1

I keep randomly getting Bad Gateway with my development configuration. It only started when I added a new service "latest". Here is my Dockerfile.latest:

# Use Python Alpine as the base image
FROM python:3.11-alpine

# Setting our work directory
WORKDIR /usr/src

# Installing necessary packages using apk, the Alpine package manager
RUN apk add --no-cache pkgconfig build-base libffi-dev python3-dev bash

# Copying our requirements file first
COPY requirements.txt /usr/src

# Install all python requirements
RUN pip3 install -r requirements.txt --no-cache-dir

# Copying all our files from our current directory to the container
COPY ./src/services/latest /usr/src
COPY ./src/config /usr/src/config

# Expose the port the app runs in
EXPOSE 8008

# Set the default command for the container
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8008"]

And here is my docker-compose.yml:

version: '3'
services:
  latest-cache:
    image: mydomain-latest-cache:v3-dev
    build:
      context: .
      dockerfile: src/services/database/Dockerfile.redis.latest
    restart: always
    command: ["redis-server", "--port", "6382", "--requirepass", "${LATEST_CACHE_REDIS_PASSWORD}"]
    ports:
      - "6382:6382"
    volumes:
      - latest_cache:/latest_cache
    labels:
      - traefik.enable=false
    networks:
      - proxy
  latest:
    image: mydomain-latest:v3-dev
    build:
      context: .
      dockerfile: src/services/latest/Dockerfile.latest
    restart: always
    ports:
      - "8008:8008"
    depends_on:
      - latest-cache
    networks:
      - proxy
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.latest.rule=Host(`localhost`) && PathPrefix(`/latest`)"
      - "traefik.http.routers.latest.entrypoints=web"
      - "traefik.http.routers.latest.middlewares=latest-stripprefix"
      - "traefik.http.middlewares.latest-stripprefix.stripprefix.prefixes=/latest"
      - "traefik.http.services.latest.loadbalancer.server.port=8008"
    environment:
      - MODE=dev
  upload:
    image: mydomain-upload:v3-dev
    build:
      context: .
      dockerfile: src/services/upload/Dockerfile.upload
    restart: always
    ports:
      - "8004:8004"
    networks:
      - proxy
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.upload.rule=Host(`localhost`) && PathPrefix(`/upload`)"
      - "traefik.http.routers.upload.entrypoints=web"
      - "traefik.http.routers.upload.middlewares=cors,oauth2-proxy"
      - "traefik.http.services.upload.loadbalancer.server.port=8004"
      - "traefik.http.middlewares.upload-cors.headers.accessControlAllowMethods=POST, OPTIONS"
      - "traefik.http.middlewares.upload-cors.headers.accessControlAllowOriginList=*"
      - "traefik.http.middlewares.upload-cors.headers.accessControlAllowHeaders=*"
      - "traefik.http.middlewares.upload-cors.headers.accessControlAllowCredentials=true"
      - "traefik.http.routers.upload.middlewares=upload-cors"
    environment:
      - MODE=dev
  oauth2-proxy:
    image: quay.io/oauth2-proxy/oauth2-proxy:v7.4.0
    networks:
      - proxy
    command:
      - --provider=oidc
      - --email-domain=*
      - --oidc-issuer-url=https://accounts.google.com
      - --cookie-secure=false
      - --cookie-secret=COOKIE_SECRET
      - --client-id=CLIENT_ID
      - --client-secret=CLIENT_SECRET
      - --upstream=http://traefik:80
      - --pass-access-token=true
      - --pass-authorization-header=true
      - --set-authorization-header=true
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.oauth2-proxy.rule=Host(`localhost`)"
      - "traefik.http.routers.oauth2-proxy.entrypoints=web"
      - "traefik.http.services.oauth2-proxy.loadbalancer.server.port=4180"
      - "traefik.http.middlewares.oauth2-proxy.forwardauth.address=http://oauth2-proxy:4180"
      - "traefik.http.middlewares.oauth2-proxy.forwardauth.trustForwardHeader=true"
    ports:
      - 4180:4180
  nginx:
    image: mydomain-nginx:v3-dev
    build:
      context: .
      dockerfile: src/static/Dockerfile.nginx.development
    restart: always
    networks:
      - proxy
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.nginx.rule=Host(`localhost`)"
      - "traefik.http.routers.nginx.entrypoints=web"
  traefik:
    image: traefik:v3.0
    restart: always
    depends_on:
      - oauth2-proxy
      - nginx
      - upload
      - latest
    ports:
      - "80:80"
      - "443:443"
      - "8080:8080"
    networks:
      - proxy
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./log:/var/log
    command:
      - "--api.insecure=true"
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--entrypoints.web.address=:80"
      - "--log.filepath=/var/log/traefik.log"
      - "--accesslog=true"
      - "--accesslog.filepath=/path-to-your/access.log"
      - "--metrics.prometheus=true"
      - "--tracing.jaeger=true"

networks:
  proxy:
    name: proxy

volumes:
  latest_cache:

Also as a suggestive side note:
I think it would be really helpful if instead of giving "Bad Gateway" on a blank screen, the devs change the behavior to output a log or something more useful. This is insanely painful. There's nothing in the dashboard and nothing in my traefik or docker logs showing why I'm getting a Bad Gateway. Please for the love of Euler, fix this!

2

You use Host() with the same domain multiple times, that does not work.

Usually bad gateway happens when multiple Docker networks are attached to the target service, but docker.network is not set.

Another occasional issue is wrong MTU when using Docker Swarm overlay network over VLAN/VSWITCH.

3

What do you mean by this?
Also am I missing something with my docker network setup?

4

You use this in multiple labels (without PathPrefix)