Can't figure why Traefik serve default SSL cert for TCP TLS passthrough

Traefik Traefik v2 (latest)
, ,
1

I'm trying to figure out why traefik serve default ssl certificate for tcp even tls.passthrough is configured.

docker-compose.yaml:

version: "3"

services:

  mariadb:
    image: mariadb:10
    restart: always
    environment:
      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
      MYSQL_USER: ${DB_USERNAME}
      MYSQL_PASSWORD: ${DB_PASSWORD}
      MYSQL_DATABASE: ${DB_DATABASE}
    volumes:
      - "mariadb-data:/var/lib/mysql"
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 512M
        reservations:
          cpus: '0.5'
          memory: 150M

  redis:
    image: redis:5-alpine
    restart: always
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 256M
        reservations:
          cpus: '0.1'
          memory: 50M

  traefik:
    image: "traefik:v2.9"
    restart: always
    command:
      - --log.level=DEBUG
      - --api=true
      - --api.dashboard=true
      - --providers.docker
      - --providers.docker.exposedByDefault=false
      - --entrypoints.web.address=:80
      - --entrypoints.web.http.redirections.entryPoint.to=websecure
      - --entrypoints.web.http.redirections.entryPoint.scheme=https
      - --entryPoints.websecure.address=:443
      - --entrypoints.websecure.http.tls=true
      # Mumble entrypoint
      - --entryPoints.mumbletcp.address=:64738
      - --entryPoints.mumbleudp.address=:64738/udp
      # ACME
      - --certificatesresolvers.primary.acme.email=${TRAEFIK_ACME_EMAIL}
      - --certificatesresolvers.primary.acme.storage=acme.json
      - --certificatesresolvers.primary.acme.httpchallenge.entrypoint=web
    ports:
      - ${TRAEFIK_HTTP_PORT}:80
      - ${TRAEFIK_HTTPS_PORT}:443
      - 64738:64738
      - 64738:64738/udp
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./acme/acme.json:/acme.json
    labels:
      - "traefik.enable=true"
      #- "traefik.http.routers.api.rule=Host(`traefik.${TRAEFIK_DOMAIN}`)"
      #- "traefik.http.routers.api.service=api@internal"
      #- "traefik.http.routers.api.entrypoints=websecure"
      - "traefik.http.routers.dashboard.entrypoints=websecure"
      - "traefik.http.routers.dashboard.rule=Host(`traefik.${TRAEFIK_DOMAIN}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
      - "traefik.http.routers.dashboard.service=api@internal"
      - "traefik.http.routers.dashboard.middlewares=auth"
      - "traefik.http.middlewares.auth.basicauth.users=user:$$2y$$10$$9uhdfyubKRCy09ezQgeCguKfvgwI3yu0RdPxBPYYILGQkHhnGhvfe"

      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.api.tls.certResolver=primary"
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 96M
        reservations:
          cpus: '0.25'
          memory: 32M

  seat-web:
    image: eveseat/seat:4
    
    restart: always
    command: web
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.seat-web.rule=Host(`${SEAT_SUBDOMAIN}.${TRAEFIK_DOMAIN}`)"
      - "traefik.http.routers.seat-web.entrypoints=websecure"
      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.seat-web.tls.certResolver=primary"
    depends_on:
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      replicas: 1
      resources:
        limits:
          cpus: '0.50'
          memory: 192M
        reservations:
          cpus: '0.1'
          memory: 90M

  seat-worker:
    image: eveseat/seat:4
    restart: always
    command: worker
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    depends_on:
      - seat-web # so that we can get db migrations done
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: '1'
          memory: 512M
        reservations:
          cpus: '0.5'
          memory: 256M

  seat-cron:
    image: eveseat/seat:4
    restart: always
    command: cron
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    depends_on:
      - seat-web # so that we can get db migrations done
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 192M
        reservations:
          cpus: '0.1'
          memory: 16M
  mumble-agent:
    image: alliancewaw/seat-mumble-register
    restart: unless-stopped
    depends_on:
      - traefik
      - mumble-server
    networks:
      - seat-network
    volumes:
      - ./mumble-agent:/app/data
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.mumble-agent.rule=Host(`mumble-agent.${TRAEFIK_DOMAIN}`)"
      - "traefik.http.routers.mumble-agent.entrypoints=websecure"
      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.mumble-agent.tls.certResolver=primary"
  mumble-server:
    image: mumblevoip/mumble-server:latest
    restart: on-failure
    depends_on:
      - traefik
    volumes:
      - ./mumble/data:/data
      - ./mumble/ssl:/ssl
      - ./mumble/secrets:/run/secrets
    networks:
      - seat-network
    labels:
      - "traefik.enable=true"
      #- "traefik.tags=service"
      - "traefik.tcp.routers.mumble-server.entrypoints=mumbletcp"
      - "traefic.tcp.routers.mumble-server.rule=HostSNI(mumble.${TRAEFIK_DOMAIN})"
      - "traefik.tcp.routers.mumble-server.tls.passthrough=true"
      - "traefik.tcp.services.mumble-server.loadbalancer.server.port=64738"
      - "traefic.udp.routers.mumble-server.entrypoints=mumbleudp"
      - "traefik.upd.services.mumble-server.loadbalancer.server.port=64738"
    environment:
      MUMBLE_SUPERUSER_PASSWORD: ${MUMBLE_SUPERUSER_PASSWORD}
      MUMBLE_CONFIG_BANDWIDTH: 72000
      MUMBLE_CONFIG_SERVER_PASSWORD: ${MUMBLE_SERVER_PASSWORD}
      MUMBLE_CONFIG_HOST: "0.0.0.0"
      MUMBLE_CONFIG_ICE: "tcp -h 0.0.0.0 -p 6502"
      MUMBLE_CONFIG_ICE_SECRET_READ: ${MUMBLE_CONFIG_ICE_SECRET_READ}
      MUMBLE_CONFIG_ICE_SECRET_WRITE: ${MUMBLE_CONFIG_ICE_SECRET_WRITE}
      MUMBLE_VERBOSE: true
    expose:
      - 6502
      - 64738
      - 64738/udp

volumes:
  mariadb-data:

networks:
  seat-network:

traefik logs:

level=debug msg="Serving default certificate for request: \"mumble.your.site\""

Folks do you have any idea why I get default cert even if I configured TLS passthrough.

2

My guess is that with your TLS.passthrough and HostSNI (with domain, you are missing backticks) you enable TLS on the entrypoint, then Traefik needs a TLS cert. As you have not assigned LE, Traefik will create a custom cert.

Should mumble handle full TLS? Then try HostSNI(*) (with backticks). For this Traefik should not need to look into the TCP TLS stream and not need a cert. But that’s just an assumption from my side.

3

I've updated missing backticks in HostSNI related to mumble and issue still exist.
But for mumble I want TLS passthrough, mumble will handle all stuff related to TLS.

Tried all things that you described, still no success. Always serve default cert.

All other traffic HTTPS serve with correct lets encrypt certificates.

latest docker-compose.yaml:

version: "3"

services:

  mariadb:
    image: mariadb:10
    restart: always
    environment:
      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
      MYSQL_USER: ${DB_USERNAME}
      MYSQL_PASSWORD: ${DB_PASSWORD}
      MYSQL_DATABASE: ${DB_DATABASE}
    volumes:
      - "mariadb-data:/var/lib/mysql"
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 512M
        reservations:
          cpus: '0.5'
          memory: 150M

  redis:
    image: redis:5-alpine
    restart: always
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 256M
        reservations:
          cpus: '0.1'
          memory: 50M

  traefik:
    image: "traefik:v2.9"
    restart: always
    command:
      - --log.level=DEBUG
      - --api=true
      - --api.dashboard=true
      - --providers.docker
      - --providers.docker.exposedByDefault=false
      - --entrypoints.web.address=:80
      - --entrypoints.web.http.redirections.entryPoint.to=websecure
      - --entrypoints.web.http.redirections.entryPoint.scheme=https
      - --entryPoints.websecure.address=:443
      - --entrypoints.websecure.http.tls=true
      # Mumble entrypoint
      - --entryPoints.mumbletcp.address=:64738
      - --entryPoints.mumbleudp.address=:64738/udp
      # ACME
      - --certificatesresolvers.primary.acme.email=${TRAEFIK_ACME_EMAIL}
      - --certificatesresolvers.primary.acme.storage=acme.json
      - --certificatesresolvers.primary.acme.httpchallenge.entrypoint=web
    ports:
      - ${TRAEFIK_HTTP_PORT}:80
      - ${TRAEFIK_HTTPS_PORT}:443
      - 64738:64738
      - 64738:64738/udp
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./acme/acme.json:/acme.json
    labels:
      - "traefik.enable=true"
      #- "traefik.http.routers.api.rule=Host(`traefik.${TRAEFIK_DOMAIN}`)"
      #- "traefik.http.routers.api.service=api@internal"
      #- "traefik.http.routers.api.entrypoints=websecure"
      - "traefik.http.routers.dashboard.entrypoints=websecure"
      - "traefik.http.routers.dashboard.rule=Host(`traefik.${TRAEFIK_DOMAIN}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
      - "traefik.http.routers.dashboard.service=api@internal"
      - "traefik.http.routers.dashboard.middlewares=auth"
      - "traefik.http.middlewares.auth.basicauth.users=user:$$2y$$10$$9uhdfyubKRCy09ezQgeCguKfvgwI3yu0RdPxBPYYILGQkHhnGhvfe"

      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.api.tls.certResolver=primary"
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 96M
        reservations:
          cpus: '0.25'
          memory: 32M

  seat-web:
    image: eveseat/seat:4
    
    restart: always
    command: web
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.seat-web.rule=Host(`${SEAT_SUBDOMAIN}.${TRAEFIK_DOMAIN}`)"
      - "traefik.http.routers.seat-web.entrypoints=websecure"
      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.seat-web.tls.certResolver=primary"
    depends_on:
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      replicas: 1
      resources:
        limits:
          cpus: '0.50'
          memory: 192M
        reservations:
          cpus: '0.1'
          memory: 90M

  seat-worker:
    image: eveseat/seat:4
    restart: always
    command: worker
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    depends_on:
      - seat-web # so that we can get db migrations done
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: '1'
          memory: 512M
        reservations:
          cpus: '0.5'
          memory: 256M

  seat-cron:
    image: eveseat/seat:4
    restart: always
    command: cron
    #    volumes:
    #      - ./packages:/var/www/seat/packages:ro  # development only
    env_file:
      - .env
    depends_on:
      - seat-web # so that we can get db migrations done
      - mariadb
      - redis
    networks:
      - seat-network
    logging:
      driver: "json-file"
      options:
        max-size: "10Mb"
        max-file: "5"
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 192M
        reservations:
          cpus: '0.1'
          memory: 16M
  mumble-agent:
    image: alliancewaw/seat-mumble-register
    restart: unless-stopped
    depends_on:
      - traefik
      - mumble-server
    networks:
      - seat-network
    volumes:
      - ./mumble-agent:/app/data
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.mumble-agent.rule=Host(`mumble-agent.${TRAEFIK_DOMAIN}`)"
      - "traefik.http.routers.mumble-agent.entrypoints=websecure"
      # Uncomment to enable HTTPS via ACME
      - "traefik.http.routers.mumble-agent.tls.certResolver=primary"
  mumble-server:
    image: mumblevoip/mumble-server:latest
    restart: on-failure
    depends_on:
      - traefik
    volumes:
      - ./mumble/data:/data
      - ./mumble/ssl:/ssl
      - ./mumble/secrets:/run/secrets
    networks:
      - seat-network
    labels:
      - "traefik.enable=true"
      #- "traefik.tags=service"
      - "traefik.tcp.routers.mumble-server.entrypoints=mumbletcp"
      - "traefic.tcp.routers.mumble-server.rule=HostSNI(`*`)"
      - "traefik.tcp.routers.mumble-server.tls=true"
      - "traefik.tcp.routers.mubmle-server.tls.certresolver=primary"
      - "traefik.tcp.routers.mumble-server.tls.passthrough=true"
      - "traefik.tcp.services.mumble-server.loadbalancer.server.port=64738"
      - "traefic.udp.routers.mumble-server.entrypoints=mumbleudp"
      - "traefik.upd.services.mumble-server.loadbalancer.server.port=64738"
    environment:
      MUMBLE_SUPERUSER_PASSWORD: ${MUMBLE_SUPERUSER_PASSWORD}
      MUMBLE_CONFIG_BANDWIDTH: 72000
      MUMBLE_CONFIG_SERVER_PASSWORD: ${MUMBLE_SERVER_PASSWORD}
      MUMBLE_CONFIG_HOST: "0.0.0.0"
      MUMBLE_CONFIG_ICE: "tcp -h 0.0.0.0 -p 6502"
      MUMBLE_CONFIG_ICE_SECRET_READ: ${MUMBLE_CONFIG_ICE_SECRET_READ}
      MUMBLE_CONFIG_ICE_SECRET_WRITE: ${MUMBLE_CONFIG_ICE_SECRET_WRITE}
      MUMBLE_VERBOSE: true
  # ports:
      #- 64738:64738
      #- 64738:64738/udp
    expose:
      - 6502
      - 64738
      - 64738/udp

volumes:
  mariadb-data:

networks:
  seat-network:
4

With this you enable TLS in Traefik. Traefik will use an existing certificate, create one with certresolver or fallback to a default cert. Remove it all to have a plain TCP forward, so mumble can handle everything.

The passthrough means that Traefik will pass on the encrypted traffic, but traffic that it can decrypt with it's own certificate.

On second thought: this will probably not work because LetsEncrypt only works with ports 80 (httpChallenge) and 443 (tlsChallenge). So if mumble runs on a different port, it can only work with more complicated dnsChallenge. Otherwise it can only create a custom certificate, which is not trusted by the browser and needs a manual import.

On third thought: you can enable Traefik TLS and have mumble with it's own custom certificate only internally, use insecureskipverify to let Traefik forward requests to a custom cert target.

5

Still In logs I don't see any problem to create cert for mumble
Why traefik try default certificate when tls.passthrough is enabled?

6

Find the typo (hint: mubmle).

You enable TLS, so Traefik needs a cert. With LetsEncrypt httpChallenge you can’t create a wildcard cert. Try full domain name again after fixing the typo.

7

Hi! I've fixed typo, add correct host to get valid cert and still traefik serve default cert.
logs ACME call looks good, without error.

8

This is the point where you expect some magic to happen, that your problem is suddenly resolved? :laughing:

Otherwise I suggest you post your current Traefic static and dynamic config again, and docker-compose.yml. Potentially add some debug log output.