Unable to generate Let's Encrypt certificates

Traefik Traefik v2
,
1

Hello, I'm trying to generate new LE certificates for my domain via Traefik. I'll post an excerpt of my Traefik logs and my configuration files.
I checked that both my ports 80 and 443 are open and reaching the server. I also cleared the acme.json file and I'm not sure what else to try. The HTTP-01 challenge used to work for me before and I haven't touched my configs in months I believe, so unless something has changed with the latest Traefik versions, I don't know what the issue could be. I've seen one error references a "/.well-known/" folder but this is the first time I've ever heard of it, just a hunch, maybe it's normal.

Any ideas? Tell me if you need more info, thanks.

docker logs -f traefik

level=debug msg="Building ACME client..." providerName=le-http.acme
level=debug msg="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=le-http.acme
level=info msg=Register... providerName=le-http.acme
level=debug msg="legolog: [INFO] acme: Registering account for mypersonal@email.com"
level=debug msg="Using HTTP Challenge provider." providerName=le-http.acme
level=debug msg="legolog: [INFO] [traefik.example.com] acme: Obtaining bundled SAN certificate"
level=debug msg="legolog: [INFO] [bw.example.com] acme: Obtaining bundled SAN certificate"
level=debug msg="legolog: [INFO] [traefik.example.com] AuthURL: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627950"
level=debug msg="legolog: [INFO] [traefik.example.com] acme: Could not find solver for: tls-alpn-01"
level=debug msg="legolog: [INFO] [traefik.example.com] acme: use http-01 solver"
level=debug msg="legolog: [INFO] [traefik.example.com] acme: Trying to solve HTTP-01"
level=debug msg="legolog: [INFO] [bw.example.com] AuthURL: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627952"
level=debug msg="legolog: [INFO] [bw.example.com] acme: Could not find solver for: tls-alpn-01"
level=debug msg="legolog: [INFO] [bw.example.com] acme: use http-01 solver"
level=debug msg="legolog: [INFO] [bw.example.com] acme: Trying to solve HTTP-01"
level=debug msg="Serving default certificate for request: \"traefik.example.com\""
level=debug msg="Serving default certificate for request: \"bw.example.com\""

level=debug msg="vulcand/oxy/roundrobin/rr: begin ServeHttp on request" Request="{\"[...lots of stuff here...]

level=debug msg="legolog: [INFO] Deactivating auth: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627950"
level=debug msg="legolog: [INFO] Unable to deactivate the authorization: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627950"
level=error msg="Unable to obtain ACME certificate for domains \"traefik.example.com\": unable to generate a certificate for the domains [traefik.example.com]: error: one or more domains had a problem:\n[traefik.example.com] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: Invalid response from https://traefik.example.com/.well-known/acme-challenge/_LNFxTT5H7qcR4CtQ-7Q0sQGZNtSaTvbb17GjIxlOs4 [MY_IP]: 404, url: \n" providerName=le-http.acme routerName=dashboard-https@file rule="Host(`traefik.example.com`)"
level=debug msg="legolog: [INFO] Deactivating auth: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627952"
level=debug msg="legolog: [INFO] Unable to deactivate the authorization: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/200627952"
level=error msg="Unable to obtain ACME certificate for domains \"bw.example.com\": unable to generate a certificate for the domains [bw.example.com]: error: one or more domains had a problem:\n[bw.example.com] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: Invalid response from https://bw.example.com/.well-known/acme-challenge/atNYlgD8G85vVZgfc2Zz2fGy63cFVeZqIwJs8i0bpyg [MY_IP]: \"\\n            <!DOCTYPE html>\\n            <html lang=\\\"en\\\">\\n            <head>\\n                <meta charset=\\\"utf-8\\\">\\n            \", url: \n" providerName=le-http.acme routerName=bitwarden-ui-https@docker rule="Host(`bw.example.com`)"

docker-compose.yml

version: '3.3'

services:
  traefik:
    container_name: traefik
    image: traefik:latest
    restart: always
    security_opt:
      - no-new-privileges:true

    ports:
      - 80:80
      - 443:443

    networks:
      - traefik

    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./traefik/acme.json:/acme.json
      - ./traefik/traefik.yml:/traefik.yml:ro
      - ./traefik/dynamic.yml:/dynamic.yml:ro

  bitwarden:
    image: bitwardenrs/server:latest
    container_name: bitwarden
    restart: always

    volumes:
      - ./bitwarden/bw-data:/data

    environment:
      - DISABLE_ADMIN_TOKEN=false
      - ADMIN_TOKEN=token
      - SIGNUPS_ALLOWED=false
      - WEBSOCKET_ENABLED=true
      - INVITATIONS_ALLOWED=false
      - DOMAIN=https://bw.example.com

    networks:
      - traefik

    labels:
      - traefik.enable=true
      - traefik.docker.network=traefik
      - traefik.http.routers.bitwarden-ui-http.rule=Host(`bw.example.com`)
      - traefik.http.routers.bitwarden-ui-http.entrypoints=http
      - traefik.http.routers.bitwarden-ui-http.service=bitwarden-ui
      - traefik.http.routers.bitwarden-ui-https.rule=Host(`bw.example.com`)
      - traefik.http.routers.bitwarden-ui-https.entrypoints=https
      - traefik.http.routers.bitwarden-ui-https.service=bitwarden-ui
      - traefik.http.routers.bitwarden-ui-https.tls=true
      - traefik.http.routers.bitwarden-ui-https.tls.certresolver=le-http
      - traefik.http.services.bitwarden-ui.loadbalancer.server.port=80
      - traefik.http.routers.bitwarden-websocket-http.rule=Host(`bw.example.com`) && Path(`/notifications/hub`)
      - traefik.http.routers.bitwarden-websocket-http.entrypoints=http
      - traefik.http.routers.bitwarden-websocket-http.service=bitwarden-websocket
      - traefik.http.routers.bitwarden-websocket-https.rule=Host(`bw.example.com`) && Path(`/notifications/hub`)
      - traefik.http.routers.bitwarden-websocket-https.entrypoints=https
      - traefik.http.routers.bitwarden-websocket-https.service=bitwarden-websocket
      - traefik.http.routers.bitwarden-websocket-https.tls=true
      - traefik.http.routers.bitwarden-websocket-https.tls.certresolver=le-http
      - traefik.http.services.bitwarden-websocket.loadbalancer.server.port=3012

networks:
  traefik:
    external: true

traefik.yml

log:
  level: DEBUG

api:
  dashboard: true
  insecure: false
  debug: false

entryPoints:
  http:
    address: ":80"
    http:
      middlewares:
      - https-redirect@file

  https:
    address: ":443"
    http:
      middlewares:
        - secureHeaders@file
      tls:
        certResolver: le-http

providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    network: traefik
    exposedByDefault: false

  file:
    filename: /dynamic.yml
    watch: true

certificatesResolvers:
  le-http:
    acme:
      email: mypersonal@email.com
      storage: acme.json
      caServer: "https://acme-staging-v02.api.letsencrypt.org/directory"
#      caServer: "https://acme-v02.api.letsencrypt.org/directory"
      httpChallenge:
        entryPoint: http

  le-dns:
    acme:
      email: mypersonal@email.com
      storage: acme.json
#      caServer: "https://acme-staging-v02.api.letsencrypt.org/directory"
      caServer: "https://acme-v02.api.letsencrypt.org/directory"
      dnsChallenge:
        provider: cloudflare
        delayBeforeCheck: 0
        resolvers:
          - "1.1.1.1:53"
          - "8.8.8.8:53"

dynamic.yml

http:
  routers:
    dashboard-http:
      entryPoints:
        - http
      rule: Host(`traefik.example.com`)
      service: api@internal

    dashboard-https:
      entryPoints:
        - https
      rule: Host(`traefik.example.com`)
      middlewares:
#        - user-auth
      service: api@internal
      tls:
        options: default
        certResolver: le-http

  middlewares:
    https-redirect:
      redirectScheme:
        scheme: https
        permanent: true

    secureHeaders:
      headers:
        frameDeny: true
        sslRedirect: true
        browserXssFilter: true
        contentTypeNosniff: true
        forceSTSHeader: true
        stsIncludeSubdomains: true
        stsPreload: true
        stsSeconds: 31536000

    user-auth:
      basicAuth:
        users:
          - "user:$passwordhash"

tls:
  options:
    default:
      cipherSuites:
        - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
      minVersion: VersionTLS12

Cloudflare DNS settings

cfdns
cfdns1031×291 14.5 KB

2

I'd drop these off and use TLS-ALNP-01.

The challenge coming in looks like it is being redirected to https and HTTP-01 is bound to http.:

https://traefik.example.com/.well-known/acme-challenge/_LNFxTT5H7qcR4CtQ-7Q0sQGZNtSaTvbb17GjIxlOs4
3

Temporarily disabling the https redirect middlewares seems to have fixed the issue indeed, now I got the staging certificates correctly, thanks.
What would be the advantage of using TLS-ALPN-01 over HTTP-01? I used the DNS challenge in the past but I don't really want to mess much with the records so I switched to HTTP.

4

TLS-ALPN-01 is handles directly at the TLS handshake layer, so all you need is port 443 open.

No routers or middlewares can mess it up.

5

Sounds good, I'll try setting it up later. Thanks!

6

I'm trying to figure out how to set up the TLS-ALPN-01 challenge. I'm currently getting some errors.

docker logs -f traefik

level=debug msg="Looking for provided certificate(s) to validate [\"bw.example.com\"]..." providerName=le-tls.acme routerName=bitwarden-websocket-https@docker rule="Host(`bw.example.com`) && Path(`/notifications/hub`)"
level=debug msg="Domains [\"bw.example.com\"] need ACME certificates generation for domains \"bw.example.com\"." routerName=bitwarden-websocket-https@docker rule="Host(`bw.example.com`) && Path(`/notifications/hub`)" providerName=le-tls.acme
level=debug msg="Loading ACME certificates [bw.example.com]..." providerName=le-tls.acme routerName=bitwarden-websocket-https@docker rule="Host(`bw.example.com`) && Path(`/notifications/hub`)"
level=debug msg="Building ACME client..." providerName=le-tls.acme
level=debug msg="https://acme-v02.api.letsencrypt.org/directory" providerName=le-tls.acme
level=info msg=Register... providerName=le-tls.acme
level=debug msg="legolog: [INFO] acme: Registering account for mypersonal@email.com"
level=debug msg="Using TLS Challenge provider." providerName=le-tls.acme
msg="legolog: [INFO] [traefik.example.com] acme: Obtaining bundled SAN certificate"
msg="legolog: [INFO] [bw.example.com] acme: Obtaining bundled SAN certificate"
msg="legolog: [INFO] [traefik.example.com] AuthURL: https://acme-v02.api.letsencrypt.org/acme/authz-v3/10482475804"
msg="legolog: [INFO] [traefik.example.com] acme: use tls-alpn-01 solver"
msg="legolog: [INFO] [traefik.example.com] acme: Trying to solve TLS-ALPN-01"
msg="TLS Challenge Present temp certificate for traefik.example.com" providerName=tlsalpn.acme
level=debug msg="legolog: [INFO] [bw.example.com] AuthURL: https://acme-v02.api.letsencrypt.org/acme/authz-v3/10482475836"
level=debug msg="legolog: [INFO] [bw.example.com] acme: use tls-alpn-01 solver"
level=debug msg="legolog: [INFO] [bw.example.com] acme: Trying to solve TLS-ALPN-01"
level=debug msg="TLS Challenge Present temp certificate for bw.example.com" providerName=tlsalpn.acme
level=debug msg="Configuration received from provider tlsalpn.acme: {\"http\":{},\"tls\":{}}" providerName=tlsalpn.acme
level=debug msg="Adding certificate for domain(s) acme challenge temp,traefik.example.com"
level=debug msg="No default certificate, generating one"
level=debug msg="legolog: [INFO] Deactivating auth: https://acme-v02.api.letsencrypt.org/acme/authz-v3/10482475804"
level=error msg="Unable to obtain ACME certificate for domains \"traefik.example.com\": unable to generate a certificate for the domains [traefik.example.com]: error: one or more domains had a problem:\n[traefik.example.com] [traefik.example.com] acme: error presenting token: timeout 2021-01-30 22:21:44.75663768 +0100 CET m=+103.772320830\n" routerName=dashboard-https@file rule="Host(`traefik.example.com`)" providerName=le-tls.acme

traefik.yml updated lines

entryPoints:
  https:
    address: ":443"
    http:
      middlewares:
        - secureHeaders@file
      tls:
        certResolver: le-tls

certificatesResolvers:
  le-tls:
    acme:
      email: mypersonal@email.com
      storage: acme.json
      caServer: "https://acme-staging-v02.api.letsencrypt.org/directory"
      tlsChallenge: {}

dynamic.yml updated lines

    dashboard-https:
      entryPoints:
        - https
      rule: Host(`traefik.rpicasa1.tk`)
      middlewares:
        - user-auth
      service: api@internal
      tls:
        options: default
        certResolver: le-tls

Should the "tlsChallenge: {}" curly braces be empty? I've checked the documentation but there's not much written about it.

7

Yes, that is what is says in the code box at acme -> TLS Challenge

I use the command line flags, but tlsChallenge is also the default challenge:

--certificatesresolvers.<name>.acme.tlschallenge :
Activate TLS-ALPN-01 Challenge. (Default: true )

8

Thanks. I'm still not able to get it to work though, I'm getting the same errors in my logs, posted below.

I'm sure the 443 port on my router is forwarded correctly because I'm getting a response when using a port listener (nc -l -p 443 -- of course with docker stopped) but I don't get any hits while Let's Encrypt is trying to get certificates.
The DNS is pointing to the correct public IP so I'm not sure if it's something wrong with my Traefik configuration, on the other hand HTTP-01 challenge works fine over port 80.

Do you have any ideas? Do the yaml files look correct to you?
I'll recheck my network configuration to make sure but it looks fine to me. I've also tried disabling the secureHeaders middleware but it doesn't help.

docker logs -f traefik

level=debug msg="Building ACME client..." providerName=le-tls.acme
level=debug msg="https://acme-v02.api.letsencrypt.org/directory" providerName=le-tls.acme
level=info msg=Register... providerName=le-tls.acme
level=debug msg="legolog: [INFO] acme: Registering account for mypersonal@email.com"
level=debug msg="Using TLS Challenge provider." providerName=le-tls.acme
level=debug msg="legolog: [INFO] [traefik.example.com] acme: Obtaining bundled SAN certificate"
level=debug msg="No default certificate, generating one"
level=debug msg="legolog: [INFO] [traefik.example.com] AuthURL: https://acme-v02.api.letsencrypt.org/acme/authz-v3/10533585445"
level=debug msg="legolog: [INFO] [traefik.example.com] acme: use tls-alpn-01 solver"
level=debug msg="legolog: [INFO] [traefik.example.com] acme: Trying to solve TLS-ALPN-01"
level=debug msg="TLS Challenge Present temp certificate for traefik.example.com" providerName=tlsalpn.acme
level=debug msg="Configuration received from provider tlsalpn.acme: {\"http\":{},\"tls\":{}}" providerName=tlsalpn.acme
level=debug msg="legolog: [INFO] Deactivating auth: https://acme-v02.api.letsencrypt.org/acme/authz-v3/10533585445"
level=error msg="Unable to obtain ACME certificate for domains \"traefik.example.com\": unable to generate a certificate for the domains [traefik.example.com]: error: one or more domains had a problem:\n[traefik.example.com] [traefik.example.com] acme: error presenting token: timeout 2021-02-01 22:30:27.544977709 +0100 CET m=+91.576358100\n" rule="Host(`traefik.example.com`)" providerName=le-tls.acme routerName=dashboard-https@file

docker-compose.yml

version: '3.3'

services:
  traefik:
    container_name: traefik
    image: traefik:latest
    restart: always
    security_opt:
      - no-new-privileges:true

    ports:
      - 80:80
      - 443:443

    networks:
      - traefik

    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./traefik/acme.json:/acme.json
      - ./traefik/traefik.yml:/traefik.yml:ro
      - ./traefik/dynamic.yml:/dynamic.yml:ro

  bitwarden:
    image: bitwardenrs/server:latest
    container_name: bitwarden
    restart: always

    volumes:
      - ./bitwarden/bw-data:/data

    environment:
      - DISABLE_ADMIN_TOKEN=false
      - ADMIN_TOKEN=token
      - SIGNUPS_ALLOWED=false
      - WEBSOCKET_ENABLED=true
      - INVITATIONS_ALLOWED=false
      - DOMAIN=https://bw.example.com

    networks:
      - traefik

    labels:
      - traefik.enable=true
      - traefik.docker.network=traefik
      - traefik.http.routers.bitwarden-ui-http.rule=Host(`bw.example.com`)
      - traefik.http.routers.bitwarden-ui-http.entrypoints=http
      - traefik.http.routers.bitwarden-ui-http.service=bitwarden-ui
      - traefik.http.routers.bitwarden-ui-https.rule=Host(`bw.example.com`)
      - traefik.http.routers.bitwarden-ui-https.entrypoints=https
      - traefik.http.routers.bitwarden-ui-https.service=bitwarden-ui
      - traefik.http.routers.bitwarden-ui-https.tls=true
      - traefik.http.routers.bitwarden-ui-https.tls.certresolver=le-tls
      - traefik.http.services.bitwarden-ui.loadbalancer.server.port=80
      - traefik.http.routers.bitwarden-websocket-http.rule=Host(`bw.example.com`) && Path(`/notifications/hub`)
      - traefik.http.routers.bitwarden-websocket-http.entrypoints=http
      - traefik.http.routers.bitwarden-websocket-http.service=bitwarden-websocket
      - traefik.http.routers.bitwarden-websocket-https.rule=Host(`bw.example.com`) && Path(`/notifications/hub`)
      - traefik.http.routers.bitwarden-websocket-https.entrypoints=https
      - traefik.http.routers.bitwarden-websocket-https.service=bitwarden-websocket
      - traefik.http.routers.bitwarden-websocket-https.tls=true
      - traefik.http.routers.bitwarden-websocket-https.tls.certresolver=le-tls
      - traefik.http.services.bitwarden-websocket.loadbalancer.server.port=3012


networks:
  traefik:
    external: true

traefik.yml

log:
  level: DEBUG

api:
  dashboard: true
  insecure: false
  debug: false

entryPoints:
  http:
    address: ":80"
    http:
      middlewares:
      - https-redirect@file

  https:
    address: ":443"
    http:
      middlewares:
        - secureHeaders@file
      tls:
        certResolver: le-tls
 
providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"   
    network: traefik
    exposedByDefault: false

  file:
    filename: /dynamic.yml
    watch: true

certificatesResolvers:
  le-tls:
    acme:
      email: mypersonal@email.com
      storage: acme.json
      tlsChallenge: {}

  le-http:
    acme:
      email: mypersonal@email.com
      storage: acme.json
#      caServer: "https://acme-staging-v02.api.letsencrypt.org/directory"
      caServer: "https://acme-v02.api.letsencrypt.org/directory"
      httpChallenge:
        entryPoint: http

  le-dns:
    acme:
      email: mypersonal@email.com
      storage: acme.json
#      caServer: "https://acme-staging-v02.api.letsencrypt.org/directory"
      caServer: "https://acme-v02.api.letsencrypt.org/directory"
      dnsChallenge:
        provider: cloudflare
        delayBeforeCheck: 0
        resolvers:
          - "1.1.1.1:53"
          - "8.8.8.8:53"

dynamic.yml

http:
  routers:
    dashboard-http:
      entryPoints:
        - http
      rule: Host(`traefik.example.com`)
      service: api@internal

    dashboard-https:
      entryPoints:
        - https
      rule: Host(`traefik.example.com`)
      middlewares:
        - user-auth
      service: api@internal
      tls:
        options: default
        certResolver: le-tls

  middlewares:
    https-redirect:
      redirectScheme:
        scheme: https
        permanent: true

    secureHeaders:
      headers:
        frameDeny: true
        sslRedirect: true
        browserXssFilter: true
        contentTypeNosniff: true
        forceSTSHeader: true
        stsIncludeSubdomains: true
        stsPreload: true
        stsSeconds: 31536000

    user-auth:
      basicAuth:
        users:
          - "user:password"

tls:
  options:
    default:
      cipherSuites:
        - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
      minVersion: VersionTLS12
9

This topic was automatically closed 3 days after the last reply. New replies are no longer allowed.