Getting error "tls: client requested unsupported application protocols ([acme-tls/1])" with docker-swarm

I'm trying to get the let's encrypt certificate using the staging server but getting the below error in the logs.

Error

tls: client requested unsupported application protocols ([acme-tls/1])

Traefik.toml

#################
#### MIDDLEWARES
#################
[http.middlewares]
  [http.middlewares.security-headers.headers]
    accessControlAllowMethods= ["GET", "OPTIONS", "PUT"]
    #accessControlAllowOrigin = "origin-list-or-null"
    accessControlMaxAge = 100
    addVaryHeader = true
    #frameDeny = true
    #sslRedirect = true
    browserXssFilter = true
    contentTypeNosniff = true
    #
    stsIncludeSubdomains = true
    stsPreload = true
    stsSeconds = 31536000

#####################
#### CUSTOM TLS CERT
#####################

[tls]
  [tls.options]
    [tls.options.myoptions]
      minVersion = "VersionTLS12"
      curvePreferences = ["CurveP521", "CurveP384"]
      sniStrict = true
      alpnProtocols = ["http/1.1", "h2"]
      cipherSuites = [
        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", # tls1.2
        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
        #"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", # 128 bit
        "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
        "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", # tls1.2
        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
        "TLS_FALLBACK_SCSV", # Client is doing version fallback. See RFC 7507.
        "TLS_AES_256_GCM_SHA384",  # tls1.3
        "TLS_CHACHA20_POLY1305_SHA256" # tls1.3

      ]

stack-traefik.yaml

version: '3.8'

services:
  traefik:
    image: "traefik:v2.9.6"
    hostname: "traefik"
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 5
      
      labels:
        - "traefik.enable=false"
        - "traefik.docker.network=loki"
        - "traefik.http.routers.api.rule=Host(`traefik.xyz`)  && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
        - "traefik.http.routers.api.service=api@internal"
        ## Middlewares
        - "traefik.http.middlewares.auth.basicauth.users=alpha:$$1$$SEjVHN0z$$Apa3.iHJAW2dbAi6OuwDe/"
        - "traefik.http.routers.api.middlewares=security-headers@file,auth"
        # enable https for api/dashboard
        - "traefik.http.routers.api.tls.certresolver=letsencrypt"
        - "traefik.http.routers.api.entrypoints=websecure"
        - "traefik.http.routers.api.tls.domains[0].main=traefik.xyz"
        # tls options from file
        - "traefik.http.routers.api.tls.options=myoptions@file"
        # dummy port
        - "traefik.http.services.dummyservice.loadbalancer.server.port=1111" # In swarm mode, traefik requires a dummy Port

    command:
      - --api=true
      - --api.dashboard=false
      - --providers.file.filename=/etc/traefik/traefik-proxy-config.toml # Using file for reading the dynamic config
      - --providers.file.watch=true
      - --providers.docker=true
      - --providers.docker.endpoint=unix:///var/run/docker.sock
      - --providers.docker.swarmMode=true
      - --providers.docker.exposedbydefault=false
      - --log.level=DEBUG
      - --accesslog=false
      - --entryPoints.web.address=:80
      - --entrypoints.web.http.redirections.entrypoint.to=websecure
      - --entrypoints.web.http.redirections.entrypoint.scheme=https
      - --entrypoints.web.http.redirections.entrypoint.permanent=true
      - --entryPoints.websecure.address=:443
      - --certificatesResolvers.letsencrypt.acme.caServer=https://acme-staging-v02.api.letsencrypt.org/directory
      - --certificatesResolvers.letsencrypt.acme.tlsChallenge=true
      - --certificatesresolvers.letsencrypt.acme.email=abc@email.com
      - --certificatesResolvers.letsencrypt.acme.storage=/letsencrypt/acme.json
    ports:
      - target: 80
        published: 80
        mode: host

      - target: 443
        published: 443
        mode: host

    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "./letsencrypt:/letsencrypt"
      - "./traefik-proxy-config.toml:/etc/traefik/traefik-proxy-config.toml:ro"
#- "./basicauth:/basicauth:ro"
    networks:
      loki:

networks:
  loki:
    external: true
    name: loki

stack-grafana.yaml

version: "3.8"

configs:
  loki-conf:
    file: ./loki.conf

services:
  loki:
    image: grafana/loki:2.7.1
    hostname: loki
      #configs:
      # - source: loki-conf
      #  target: /etc/loki/local-config.yaml
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 5
      
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.loki.rule=PathPrefix(`/loki/`)"
        - "traefik.http.services.loki.loadbalancer.server.port=3100"
    ports:
      - target: 3100
        published: 3100
        protocol: tcp
        mode: host
    networks:
      loki:
  grafana:
    image: grafana/grafana:9.3.2
    hostname: grafana
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 5
            
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=loki"
        - "traefik.http.routers.grafana-secure.entrypoints=websecure"
        - "traefik.http.routers.grafana-secure.rule=Host(`grafana.xyz`)"
            ## Middleware
        - "traefik.http.routers.grafana-secure.middlewares=security-headers@file"
            ## LetsEncrypt
        - "traefik.http.routers.grafana-secure.tls=true"
        - "traefik.http.routers.grafana-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.grafana-secure.tls.domains[0].main=grafana.xyz"
        - "traefik.http.routers.grafana-secure.tls.options=myoptions@file"
            ## Service
        - "traefik.http.routers.grafana-secure.service=grafana" #here service name is grafana
        - "traefik.http.services.grafana.loadbalancer.server.port=3000"
        - "traefik.http.services.grafana.loadbalancer.passhostheader=true"

    ports:
      - target: 3000
        #published: 3100
        protocol: tcp

    environment:
      GF_RENDERING_SERVER_URL: http://renderer:8081/render
      GF_RENDERING_CALLBACK_URL: http://grafana:3000/
      GF_LOG_FILTERS: rendering:debug
    networks:
      loki:
  
networks:
  loki:
    external: true
    name: loki

Traefik (community edition) does not support LetsEncrypt certificate generation when using multiple Traefik instances in Docker Swarm. You need the enterprise version to do sync'ed LE cert validation. (Source)

The issue is that Traefik will start a request, gets a key from LE, but a different Traefik instance will receive the validation and not know about the key.

I created a rough workaround using certbot for LE certificate generation when using multiple Traefik instances in Docker Swarm, it might not be 100% reliable.

Alternatively use a shared acme.json through a distributed file system, but that might be problematic due to potentially multiple Traefik instances writing to the same file at the same time.