Please help me get lets encrypt working

I’ve verified that the .well known link is accessible. here is my redacted config. I see the request in logs

2025-12-19T02:58:47Z INF Testing certificate renew... acmeCA=https://acme-v02.api.letsencrypt.org/directory providerName=le.acme

and I also see script kitties hitting the .well known url

I do NOT see the request getting blocked going out of my network. I have an ubiquiti network.

services:
  traefik:
    image: traefik:v3.6.5

    networks:
      - traefik_proxy

    ports:
      - target: 80
        published: 80
        protocol: tcp
        mode: host
      - target: 443
        published: 443
        protocol: tcp
        mode: host

    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./certs:/certs:ro
      - ./dynamic:/dynamic:ro
      - letsencrypt:/letsencrypt

    command:
      - "--entrypoints.web.address=:80"
      - "--entrypoints.web.http.redirections.entrypoint.to=websecure"
      - "--entrypoints.web.http.redirections.entrypoint.scheme=https"
      - "--entrypoints.web.http.redirections.entrypoint.permanent=true"
      - "--entrypoints.websecure.address=:443"
      - "--entrypoints.websecure.http.tls=true"
      - "--providers.file.filename=/dynamic/tls.yaml"
      - "--providers.swarm.endpoint=unix:///var/run/docker.sock"
      - "--providers.swarm.watch=true"
      - "--providers.swarm.exposedbydefault=false"
      - "--providers.swarm.network=traefik_traefik_proxy"
      - "--api.dashboard=true"
      - "--api.insecure=false"
      - "--log.level=INFO"
      - "--accesslog=true"
      - "--metrics.prometheus=true"
      - "--certificatesresolvers.le.acme.email=XXXXXX@gmail.com"
      - "--certificatesresolvers.le.acme.storage=/letsencrypt/acme.json"
      - "--certificatesresolvers.le.acme.httpchallenge.entrypoint=web"
      - "--certificatesresolvers.le.acme.tlschallenge=true"
#      - "--certificatesresolvers.le.acme.caServer=https://acme-staging-v02.api.letsencrypt.org/directory"
      - "--entrypoints.websecure.http.tls.certresolver=le"

    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.dashboard.rule=Host(`dashboard.XXXXX.us`)"
        - "traefik.http.routers.dashboard.entrypoints=websecure"
        - "traefik.http.routers.dashboard.service=api@internal"
        - "traefik.http.routers.dashboard.tls=true"
        - "traefik.http.middlewares.dashboard-auth.basicauth.users=admin:$$apr1$$UuWqc6VR$$EZAJrDJ4al/NpV3p0Fn2v."
        - "traefik.http.routers.dashboard.middlewares=dashboard-auth@swarm"
        - "traefik.http.services.traefik.loadbalancer.server.port=8080"

  whoami:
    image: traefik/whoami
    networks:
      - traefik_proxy
    deploy:
      mode: replicated
      replicas: 5
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.whoami.rule=Host(`whoami.XXXXXX.us`)"
        - "traefik.http.routers.whoami.entrypoints=websecure"
        - "traefik.http.routers.whoami.tls=true"
        - "traefik.http.routers.whoami.tls.domains[0].main=whoami.XXXXXX.us"
        - "traefik.http.services.whoami.loadbalancer.server.port=80"
        - "traefik.http.routers.whoami.tls.certresolver=le"

networks:
  traefik_proxy:
    driver: overlay
    attachable: true
volumes:
  letsencrypt:

Enable Traefik debug log (doc) and Traefik access log in JSON format (doc).

You seem to enable httpChallenge and tlsChallenge at the same time. Decide for one.