External router middlewares with Authentik not working

Traefik Traefik v2
1

I am running Traefik 2.10 and have several external routers - all is good when i dont do any forwardAuth.
As soon as i try an do forwardAuth i get the default self signed certificate and get 404 on all services running behind Traefik.
I dont use CertResolver. I have my own wildcard letsencrypt in /etc/letsencrypt/ imported from another container.

Docker-compose.yml:

version: "3.7"
services:
  traefik:
    image: traefik:v2.10
    container_name: traefik
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
      - "8082:8082"
    networks:
       docker-net:

    environment:
      - TZ=XXX/XXX
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /var/docker/traefik/config:/etc/traefik/
      - /var/docker/letsencrypt/acme:/etc/letsencrypt/
      - /var/log/traefik:/var/log/traefik
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik-secure.entrypoints=https"
      - "traefik.http.routers.traefik-secure.rule=Host(`traefik.domain.tld`)"
      - "traefik.http.routers.traefik-secure.tls=true"
      - "traefik.http.routers.traefik-secure.middlewares=middlewares-authentik@file"
      - "traefik.http.routers.traefik-secure.service=api@internal"

      - "co.elastic.logs/fileset.stdout=access"
      - "co.elastic.logs/fileset.stderr=error"
      - "co.elastic.metrics/module=apache"
      - "co.elastic.metrics/metricsets=status"


networks:
  docker-net:
    external: true

Traefik.yml:

global:
  checkNewVersion: false
log:
  level: DEBUG
  filePath: "/var/log/traefik/error.log"
accessLog:
  filePath: "/var/log/traefik/access.log"
  fields:
    names:
      StartUTC: drop
defaultEntryPoints:
   - http
   - https
   - metrics
api:
  dashboard: true
  insecure: true
ping: {}
metrics:
  prometheus:
    buckets:
      - 0.1
      - 0.3
      - 1.2
      - 5.0
providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false
  file:
    directory: /etc/traefik/
    watch: true
http:
  middlewares:
    auth:
      basicAuth:
        users:
          - "XXX:$apr1$N97eBrIz$ltJHkXXXlOrme6GtLZzr1"
    middlewares-authentik:
      forwardAuth:
        address: "http://authentik_server:9000/outpost.goauthentik.io/auth/traefik"
        trustForwardHeader: true
        authResponseHeaders:
          - X-authentik-username
          - X-authentik-groups
          - X-authentik-email
          - X-authentik-name
          - X-authentik-uid
          - X-authentik-jwt
          - X-authentik-meta-jwks
          - X-authentik-meta-outpost
          - X-authentik-meta-provider
          - X-authentik-meta-app
          - X-authentik-meta-version
entryPoints:
  http:
    address: ":80"
    http:
      redirections:
        entryPoint:
          to: https
          scheme: https
  https:
    address: ":443"
  metrics:
    address: ":8082"
tls:
  certificates:
    - certFile: "/etc/letsencrypt/fullchain.pem"
      keyFile: "/etc/letsencrypt/private.key"
  stores:
    default:
      defaultCertificate:
        certFile: "/etc/letsencrypt/fullchain.pem"
        keyFile: "/etc/letsencrypt/private.key"
  options:
    default:
      minVersion: VersionTLS12
      cipherSuites:
        - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
serversTransport:
  insecureSkipVerify: true

Rules.yml:

http:
  routers:
    router-zig2mqtt:
      entryPoints:
        - https
      rule: Host(`zig2mqtt.domain.tld`)
      middleswares:
        middlewares-authentik:
          - forwardAuth
      tls: {}
      service: service-zig2mqtt
  services:
    service-zig2mqtt:
      loadBalancer:
        servers:
        - url: "http://X.X.X.Y:8080/"
        passHostHeader: true

Error.log:

tail -f /var/log/traefik/error.log  
time="2024-01-31T22:42:17+01:00" level=debug msg="Serving default certificate for request: \"authentik.domain.tld\""
time="2024-01-31T22:42:17+01:00" level=debug msg="Serving default certificate for request: \"authentik.domain.tld\""
time="2024-01-31T22:42:17+01:00" level=debug msg="http: TLS handshake error from 172.16.1.1:47622: remote error: tls: bad certificate"
time="2024-01-31T22:42:17+01:00" level=debug msg="http: TLS handshake error from 172.16.1.1:47638: remote error: tls: bad certificate"
time="2024-01-31T22:42:38+01:00" level=debug msg="Serving default certificate for request: \"authentik.domain.tld\""
time="2024-01-31T22:42:38+01:00" level=debug msg="http: TLS handshake error from 172.16.1.1:50068: remote error: tls: bad certificate"
time="2024-01-31T22:44:46+01:00" level=debug msg="Serving default certificate for request: \"authentik.domain.tld\""
time="2024-01-31T22:44:46+01:00" level=debug msg="http: TLS handshake error from 172.16.1.1:48848: remote error: tls: bad certificate"
time="2024-01-31T22:45:08+01:00" level=debug msg="Serving default certificate for request: \"authentik.domain.tld\""
time="2024-01-31T22:45:08+01:00" level=debug msg="http: TLS handshake error from 10.13.37.254:56860: remote error: tls: unknown certificate"
time="2024-01-31T22:46:13+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""
time="2024-01-31T22:46:13+01:00" level=debug msg="http: TLS handshake error from 10.255.255.2:54837: remote error: tls: unknown certificate"
time="2024-01-31T22:46:14+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""
time="2024-01-31T22:46:14+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""
time="2024-01-31T22:46:14+01:00" level=debug msg="http: TLS handshake error from 10.255.255.2:54839: remote error: tls: unknown certificate"
time="2024-01-31T22:46:14+01:00" level=debug msg="http: TLS handshake error from 10.255.255.2:54838: remote error: tls: unknown certificate"
time="2024-01-31T22:46:17+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""
time="2024-01-31T22:46:17+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""
time="2024-01-31T22:46:17+01:00" level=debug msg="http: TLS handshake error from 10.255.255.2:54841: remote error: tls: unknown certificate"
time="2024-01-31T22:46:17+01:00" level=debug msg="http: TLS handshake error from 10.255.255.2:54842: remote error: tls: unknown certificate"
time="2024-01-31T22:46:17+01:00" level=debug msg="Serving default certificate for request: \"zig2mqtt.domain.tld\""

Curl:

curl -k -v https://zig2mqtt.domain.tld
*   Trying 10.13.37.4:443...
* Connected to zig2mqtt2.domain.tld (10.13.37.4) port 443 (#0)
* ALPN: offers h2,http/1.1
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
* TLSv1.3 (IN), TLS handshake, Certificate (11):
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
* TLSv1.3 (IN), TLS handshake, Finished (20):
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
* TLSv1.3 (OUT), TLS handshake, Finished (20):
* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256
* ALPN: server accepted h2
* Server certificate:
*  subject: CN=TRAEFIK DEFAULT CERT
*  start date: Jan 31 21:40:33 2024 GMT
*  expire date: Jan 30 21:40:33 2025 GMT
*  issuer: CN=TRAEFIK DEFAULT CERT
*  SSL certificate verify result: self-signed certificate (18), continuing anyway.
* using HTTP/2
* h2h3 [:method: GET]
* h2h3 [:path: /]
* h2h3 [:scheme: https]
* h2h3 [:authority: zig2mqtt.domain.tld]
* h2h3 [user-agent: curl/7.88.1]
* h2h3 [accept: */*]
* Using Stream ID: 1 (easy handle 0x55629523cc80)
> GET / HTTP/2
> Host: zig2mqtt.domain.tld
> user-agent: curl/7.88.1
> accept: */*
> 
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
< HTTP/2 404 
< content-type: text/plain; charset=utf-8
< x-content-type-options: nosniff
< content-length: 19
< date: Wed, 31 Jan 2024 21:46:45 GMT
< 
404 page not found
2

Is the static config accepted? I don't see defaultEntryPoints in the static or dynamic reference (doc, doc).

middlewares and tls belongs in a dynamic config file. You just read the static file again with providers.file to load dynamic config pieces, kind of un-clean :wink:

And

curl -k -v https://zig2mqtt2mqtt.domain.tld

will never match

                  Host(`zig2mqtt.domain.tld`)

Where is authentik.domain.tld coming from? It seems nowhere defined. Is that a http redirect from the authentik service and you have it in your DNS? You would need labels with Host() on the service, too, so LE can create a cert.

3

I see i slipped in some typos in the curl example, it should ofcourse be zig2mqtt.domain.tld.
I moved almost everything from static to dynamic and it suddenly started working as expected.

But i wonder if this is "best practice" or is there anything you see straight away that you would have changed?

Rules.yml 
http:
  middlewares:
    middlewares-authentik:
      forwardAuth:
        address: "http://authentik_server:9000/outpost.goauthentik.io/auth/traefik"
        tls:
          cert: "/etc/letsencrypt/fullchain.pem"
          key: "/etc/letsencrypt/private.key"
          insecureSkipVerify: true
        trustForwardHeader: true
        authResponseHeaders:
          - X-authentik-username
          - X-authentik-groups
          - X-authentik-email
          - X-authentik-name
          - X-authentik-uid
          - X-authentik-jwt
          - X-authentik-meta-jwks
          - X-authentik-meta-outpost
          - X-authentik-meta-provider
          - X-authentik-meta-app
          - X-authentik-meta-version

## === ##

  routers:
    router-zig:
      entryPoints:
        - https
      rule: Host(`zig2mqtt.domain.tld`)
      middlewares:
        - middlewares-authentik
      tls: {}
      service: service-zig

## === ##

  services:
    service-zig:
      loadBalancer:
        servers:
          - url: "http://10.13.37.9:8080/"
        passHostHeader: true

## === ##

tls:
  certificates:
    - certFile: "/etc/letsencrypt/fullchain.pem"
      keyFile: "/etc/letsencrypt/private.key"
  stores:
    default:
      defaultCertificate:
        certFile: "/etc/letsencrypt/fullchain.pem"
        keyFile: "/etc/letsencrypt/private.key"
  options:
    default:
      minVersion: VersionTLS12
      cipherSuites:
        - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
Traefik.yml 
global:
  checkNewVersion: false
log:
  level: DEBUG
  filePath: "/var/log/traefik/error.log"
accessLog:
  filePath: "/var/log/traefik/access.log"
  fields:
    names:
      StartUTC: drop
defaultEntryPoints:
   - http
   - https
api:
  dashboard: true
#  insecure: true
  debug: true
providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false
  file:
    directory: /etc/traefik/
    watch: true
entryPoints:
  http:
    address: ":80"
    http:
      redirections:
        entryPoint:
          to: https
          scheme: https
  https:
    address: ":443"
serversTransport:
  insecureSkipVerify: true
4

I would enable tls on entrypoint https globally, not on router.

You can compare to my simple Traefik example.

And I don't think defaultEntryPoints is a thing, it's at least not in the reference.