Traefik serving default certificate instead letsencrypt

Hi, I have traefik running on proxmox in LXC container which redirects my domain to the VM, it works for me.
I have another traefik on the VM which routes the domain to docker containers, but on this traefik I can't generate letsencrypt certificate, it keeps returning TRAEFIK DEFAULT CERT. I have tried different settings, delete acme.json and restart the container I really don't know how to proceed anymore, can anyone advise me? Thank you

docker-compose.yml

version: "3.3"

services:

  traefik:
    image: "traefik:v2.9"
    container_name: "traefik"
    networks:
      - traefik-proxy
    ports:
      - "80:80"
      - "443:443"
      - "8080:8080"
    dns:
      - 8.8.8.8
      - 4.4.4.4
    volumes:
      - "./letsencrypt/acme-v4.json:/etc/traefik/acme/acme-v4.json"
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      - "./traefik.yml:/etc/traefik/traefik.yml:ro"
      - "./config.yml:/etc/traefik/config.yml:ro"
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik.entrypoints=http"
      - "traefik.http.routers.traefik.rule=Host(`monitor.dev.***`)"
      - "traefik.http.routers.traefik.service=api@internal"
      - "traefik.http.routers.traefik-secure.entrypoints=https"
      - "traefik.http.routers.traefik-secure.rule=Host(`monitor.dev.***`)"
      - "traefik.http.routers.traefik-secure.service=api@internal"
      #- "traefik.http.routers.traefik-secure.tls=true"
      - "traefik.http.routers.traefik-secure.tls.certResolver=letsEncrypt"

networks:
  traefik-proxy:
    external: true

traefik.yml

api:
  dashboard: true                             # Enable the dashboard
  insecure: true
  debug: true

log:
  level: DEBUG

# Certificate Resolvers are responsible for retrieving certificates from an ACME server
# See https://doc.traefik.io/traefik/https/acme/#certificate-resolvers
certificatesResolvers:
  letsEncrypt:
    acme:
      email: "lukas.zatloukal@foxily.cz"  # Email address used for registration
      storage: "/etc/traefik/acme/acme-v4.json"    # File or key used for certificates storage
      #tlsChallenge: true
      httpChallenge:
        entryPoint: http
      caServer: https://acme-staging-v02.api.letsencrypt.org/directory
      #caServer: https://acme-v02.api.letsencrypt.org/directory

entryPoints:
  http:
    address: ":80"                            # Create the HTTP entrypoint on port 80
    #http:
    #  redirections:                           # HTTPS redirection (80 to 443)
    #    entryPoint:
    #      to: "https"                         # The target element
    #      scheme: "https"                     # The redirection target scheme
  https:
    address: ":443"                           # Create the HTTPS entrypoint on port 443

global:
  checknewversion: true                       # Periodically check if a new version has been released.
  #sendanonymoususage: true                    # Periodically send anonymous usage statistics.

providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"   # Listen to the UNIX Docker socket
    exposedByDefault: false                   # Only expose container that are explicitly enabled (using label traefik.enabled)
    network: "traefik-proxy"                    # Default network to use for connections to all containers.
    #swarmmode: true                           # Activates the Swarm Mode (instead of standalone Docker).
    #swarmModeRefreshSeconds: 15               # Defines the polling interval (in seconds) in Swarm Mode.
    watch: true                               # Watch Docker Swarm events
  file:
    filename: "/etc/traefik/config.yml"       # Link to the dynamic configuration
    watch: true                               # Watch for modifications
  providersThrottleDuration: 10               # Configuration reload frequency

acme.json

{
  "letsEncrypt": {
    "Account": {
      "Email": "lukas.zatloukal@foxily.cz",
      "Registration": {
        "body": {
          "status": "valid",
          "contact": [
            "mailto:lukas.zatloukal@foxily.cz"
          ]
        },
        "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/100116254"
      },
      "PrivateKey": "CENSORED",
      "KeyType": "4096"
    },
    "Certificates": [
      {
        "domain": {
          "main": "monitor.dev.***"
        },
        "certificate": "CENSORED",
        "key": "CENSORED",
        "Store": "default"
      }
    ]
  }

log

traefik    | time="2023-04-29T10:16:57Z" level=info msg="Configuration loaded from file: /etc/traefik/traefik.yml"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Traefik version 2.9.4 built on 2022-10-27T18:44:34Z"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Static configuration loaded {\"global\":{\"checkNewVersion\":true},\"serversTransport\":{\"maxIdleConnsPerHost\":200},\"entryPoints\":{\"http\":{\"address\":\":80\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}},\"https\":{\"address\":\":443\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}},\"traefik\":{\"address\":\":8080\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}}},\"providers\":{\"providersThrottleDuration\":\"10s\",\"docker\":{\"watch\":true,\"endpoint\":\"unix:///var/run/docker.sock\",\"defaultRule\":\"Host(`{{ normalize .Name }}`)\",\"network\":\"traefik-proxy\",\"swarmModeRefreshSeconds\":\"15s\"},\"file\":{\"watch\":true,\"filename\":\"/etc/traefik/config.yml\"}},\"api\":{\"insecure\":true,\"dashboard\":true,\"debug\":true},\"log\":{\"level\":\"DEBUG\",\"format\":\"common\"},\"certificatesResolvers\":{\"letsEncrypt\":{\"acme\":{\"email\":\"lukas.zatloukal@foxily.cz\",\"caServer\":\"https://acme-staging-v02.api.letsencrypt.org/directory\",\"storage\":\"/etc/traefik/acme/acme-v4.json\",\"keyType\":\"RSA4096\",\"certificatesDuration\":2160,\"httpChallenge\":{\"entryPoint\":\"http\"}}}}}"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://doc.traefik.io/traefik/contributing/data-collection/\n"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider aggregator aggregator.ProviderAggregator"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Starting TCP Server" entryPointName=https
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Starting TCP Server" entryPointName=traefik
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider *file.Provider"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="*file.Provider provider configuration: {\"watch\":true,\"filename\":\"/etc/traefik/config.yml\"}"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider *traefik.Provider"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="*traefik.Provider provider configuration: {}"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider *docker.Provider"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="*docker.Provider provider configuration: {\"watch\":true,\"endpoint\":\"unix:///var/run/docker.sock\",\"defaultRule\":\"Host(`{{ normalize .Name }}`)\",\"network\":\"traefik-proxy\",\"swarmModeRefreshSeconds\":\"15s\"}"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Configuration received: {\"http\":{},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=file
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Starting TCP Server" entryPointName=http
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider *acme.ChallengeTLSALPN"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="*acme.ChallengeTLSALPN provider configuration: {}"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Starting provider *acme.Provider"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="*acme.Provider provider configuration: {\"email\":\"lukas.zatloukal@foxily.cz\",\"caServer\":\"https://acme-staging-v02.api.letsencrypt.org/directory\",\"storage\":\"/etc/traefik/acme/acme-v4.json\",\"keyType\":\"RSA4096\",\"certificatesDuration\":2160,\"httpChallenge\":{\"entryPoint\":\"http\"},\"ResolverName\":\"letsEncrypt\",\"store\":{},\"TLSChallengeProvider\":{},\"HTTPChallengeProvider\":{}}"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Attempt to renew certificates \"720h0m0s\" before expiry and check every \"24h0m0s\"" providerName=letsEncrypt.acme ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory"
traefik    | time="2023-04-29T10:16:57Z" level=info msg="Testing certificate renew..." providerName=letsEncrypt.acme ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory"
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"acme-http\":{\"entryPoints\":[\"http\"],\"service\":\"acme-http@internal\",\"rule\":\"PathPrefix(`/.well-known/acme-challenge/`)\",\"priority\":2147483647},\"api\":{\"entryPoints\":[\"traefik\"],\"service\":\"api@internal\",\"rule\":\"PathPrefix(`/api`)\",\"priority\":2147483646},\"dashboard\":{\"entryPoints\":[\"traefik\"],\"middlewares\":[\"dashboard_redirect@internal\",\"dashboard_stripprefix@internal\"],\"service\":\"dashboard@internal\",\"rule\":\"PathPrefix(`/`)\",\"priority\":2147483645},\"debug\":{\"entryPoints\":[\"traefik\"],\"service\":\"api@internal\",\"rule\":\"PathPrefix(`/debug`)\",\"priority\":2147483646}},\"services\":{\"acme-http\":{},\"api\":{},\"dashboard\":{},\"noop\":{}},\"middlewares\":{\"dashboard_redirect\":{\"redirectRegex\":{\"regex\":\"^(http:\\\\/\\\\/(\\\\[[\\\\w:.]+\\\\]|[\\\\w\\\\._-]+)(:\\\\d+)?)\\\\/$\",\"replacement\":\"${1}/dashboard/\",\"permanent\":true}},\"dashboard_stripprefix\":{\"stripPrefix\":{\"prefixes\":[\"/dashboard/\",\"/dashboard\"]}}},\"serversTransports\":{\"default\":{\"maxIdleConnsPerHost\":200}}},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=internal
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Configuration received: {\"http\":{},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Provider connection established with docker 20.10.22 (API 1.41)" providerName=docker
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Filtering disabled container" providerName=docker container=portainer-1961e2ade1b1171397b58b8fc23dbad8c30a5806e0d983cf98d9c4037cd5ea7a
traefik    | time="2023-04-29T10:16:57Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"traefik\":{\"entryPoints\":[\"http\"],\"service\":\"api@internal\",\"rule\":\"Host(`monitor.dev.**`)\“},\“traefik-secure\":{\"entryPoints\":[\"https\"],\"service\":\"api@internal\",\"rule\":\"Host(`monitor.dev.***`)\“,\“tls\":{\"certResolver\":\"letsEncrypt\"}}},\"services\":{\"traefik-traefik\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.20.0.2:80\"}],\"passHostHeader\":true}}}},\"tcp\":{},\"udp\":{}}" providerName=docker
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" middlewareName=tracing entryPointName=traefik routerName=api@internal middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware dashboard@internal" entryPointName=traefik routerName=dashboard@internal middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" entryPointName=traefik routerName=dashboard@internal middlewareType=StripPrefix middlewareName=dashboard_stripprefix@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Adding tracing to middleware" entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_stripprefix@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" routerName=dashboard@internal middlewareType=RedirectRegex middlewareName=dashboard_redirect@internal entryPointName=traefik
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Setting up redirection from ^(http:\\/\\/(\\[[\\w:.]+\\]|[\\w\\._-]+)(:\\d+)?)\\/$ to ${1}/dashboard/" middlewareType=RedirectRegex middlewareName=dashboard_redirect@internal entryPointName=traefik routerName=dashboard@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Adding tracing to middleware" routerName=dashboard@internal entryPointName=traefik middlewareName=dashboard_redirect@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" routerName=debug@internal middlewareName=tracing middlewareType=TracingForwarder entryPointName=traefik
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" middlewareType=Recovery entryPointName=traefik middlewareName=traefik-internal-recovery
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware acme-http@internal" entryPointName=http routerName=acme-http@internal middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" entryPointName=http middlewareName=traefik-internal-recovery middlewareType=Recovery
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware dashboard@internal" routerName=dashboard@internal middlewareName=tracing middlewareType=TracingForwarder entryPointName=traefik
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" routerName=dashboard@internal entryPointName=traefik middlewareName=dashboard_stripprefix@internal middlewareType=StripPrefix
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Adding tracing to middleware" routerName=dashboard@internal middlewareName=dashboard_stripprefix@internal entryPointName=traefik
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" middlewareType=RedirectRegex entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_redirect@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Setting up redirection from ^(http:\\/\\/(\\[[\\w:.]+\\]|[\\w\\._-]+)(:\\d+)?)\\/$ to ${1}/dashboard/" middlewareName=dashboard_redirect@internal middlewareType=RedirectRegex entryPointName=traefik routerName=dashboard@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Adding tracing to middleware" entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_redirect@internal
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" routerName=debug@internal middlewareName=tracing middlewareType=TracingForwarder entryPointName=traefik
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" entryPointName=traefik routerName=api@internal middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" entryPointName=traefik middlewareType=Recovery middlewareName=traefik-internal-recovery
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" entryPointName=http routerName=traefik@docker middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware acme-http@internal" routerName=acme-http@internal entryPointName=http middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" entryPointName=http middlewareName=traefik-internal-recovery middlewareType=Recovery
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Added outgoing tracing middleware api@internal" middlewareType=TracingForwarder routerName=traefik-secure@docker entryPointName=https middlewareName=tracing
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Creating middleware" middlewareType=Recovery entryPointName=https middlewareName=traefik-internal-recovery
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Adding route for monitor.dev.*** with TLS options default" entryPointName=https
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Trying to challenge certificate for domain [monitor.dev.***] found in HostSNI rule" rule="Host(`monitor.dev.***`)" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme routerName=traefik-secure@docker
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Looking for provided certificate(s) to validate [\"monitor.dev.***\"]..." routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Domains [\"monitor.dev.***\"] need ACME certificates generation for domains \"monitor.dev.***\"." routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Loading ACME certificates [monitor.dev.***]..." providerName=letsEncrypt.acme routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory"
traefik    | time="2023-04-29T10:16:58Z" level=debug msg="Serving default certificate for request: \"\""
traefik    | time="2023-04-29T10:17:02Z" level=debug msg="Building ACME client..." providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:17:02Z" level=debug msg="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:17:02Z" level=info msg=Register... providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:17:02Z" level=debug msg="legolog: [INFO] acme: Registering account for lukas.zatloukal@foxily.cz"
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="Using HTTP Challenge provider." providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="legolog: [INFO] [monitor.dev.***] acme: Obtaining bundled SAN certificate"
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="legolog: [INFO] [monitor.dev.***] AuthURL: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/6290163004"
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="legolog: [INFO] [monitor.dev.***] acme: Could not find solver for: tls-alpn-01"
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="legolog: [INFO] [monitor.dev.***] acme: use http-01 solver"
traefik    | time="2023-04-29T10:17:03Z" level=debug msg="legolog: [INFO] [monitor.dev.***] acme: Trying to solve HTTP-01"
traefik    | time="2023-04-29T10:17:04Z" level=debug msg="Unable to split host and port: address monitor.dev.***: missing port in address. Fallback to request host." providerName=acme
traefik    | time="2023-04-29T10:17:04Z" level=debug msg="Retrieving the ACME challenge for monitor.dev.*** (token \"c3dNKXPdhsCEFjfj11yHcseVeZe9_mO42uV9klz-Fnc\")..." providerName=acme
traefik    | time="2023-04-29T10:17:05Z" level=debug msg="Unable to split host and port: address monitor.dev.***: missing port in address. Fallback to request host." providerName=acme
traefik    | time="2023-04-29T10:17:05Z" level=debug msg="Retrieving the ACME challenge for monitor.dev.*** (token \"c3dNKXPdhsCEFjfj11yHcseVeZe9_mO42uV9klz-Fnc\")..." providerName=acme
traefik    | time="2023-04-29T10:17:08Z" level=debug msg="Unable to split host and port: address monitor.dev.***: missing port in address. Fallback to request host." providerName=acme
traefik    | time="2023-04-29T10:17:08Z" level=debug msg="Retrieving the ACME challenge for monitor.dev.*** (token \"c3dNKXPdhsCEFjfj11yHcseVeZe9_mO42uV9klz-Fnc\")..." providerName=acme
traefik    | time="2023-04-29T10:17:11Z" level=debug msg="legolog: [INFO] [monitor.dev.***] The server validated our request"
traefik    | time="2023-04-29T10:17:11Z" level=debug msg="legolog: [INFO] [monitor.dev.***] acme: Validations succeeded; requesting certificates"
traefik    | time="2023-04-29T10:17:18Z" level=debug msg="legolog: [INFO] Wait for certificate [timeout: 30s, interval: 500ms]"
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="legolog: [INFO] [monitor.dev.***] Server responded with a certificate."
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Certificates obtained for domains [monitor.dev.***]" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)"
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Configuration received: {\"http\":{},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=letsEncrypt.acme
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Adding certificate for domain(s) monitor.dev.***"
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware api@internal" routerName=traefik@docker middlewareType=TracingForwarder middlewareName=tracing entryPointName=http
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware acme-http@internal" middlewareName=tracing middlewareType=TracingForwarder entryPointName=http routerName=acme-http@internal
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Creating middleware" middlewareName=traefik-internal-recovery middlewareType=Recovery entryPointName=http
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware api@internal" entryPointName=traefik middlewareName=tracing middlewareType=TracingForwarder routerName=debug@internal
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware api@internal" entryPointName=traefik routerName=api@internal middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware dashboard@internal" entryPointName=traefik routerName=dashboard@internal middlewareName=tracing middlewareType=TracingForwarder
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Creating middleware" entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_stripprefix@internal middlewareType=StripPrefix
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Adding tracing to middleware" entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_stripprefix@internal
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Creating middleware" routerName=dashboard@internal middlewareName=dashboard_redirect@internal middlewareType=RedirectRegex entryPointName=traefik
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Setting up redirection from ^(http:\\/\\/(\\[[\\w:.]+\\]|[\\w\\._-]+)(:\\d+)?)\\/$ to ${1}/dashboard/" middlewareName=dashboard_redirect@internal middlewareType=RedirectRegex entryPointName=traefik routerName=dashboard@internal
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Adding tracing to middleware" entryPointName=traefik routerName=dashboard@internal middlewareName=dashboard_redirect@internal
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Creating middleware" middlewareType=Recovery entryPointName=traefik middlewareName=traefik-internal-recovery
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Added outgoing tracing middleware api@internal" middlewareName=tracing middlewareType=TracingForwarder entryPointName=https routerName=traefik-secure@docker
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Creating middleware" middlewareType=Recovery entryPointName=https middlewareName=traefik-internal-recovery
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Adding route for monitor.dev.*** with TLS options default" entryPointName=https
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Trying to challenge certificate for domain [monitor.dev.***] found in HostSNI rule" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)"
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="Looking for provided certificate(s) to validate [\"monitor.dev.***\"]..." providerName=letsEncrypt.acme routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)" ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory"
traefik    | time="2023-04-29T10:17:20Z" level=debug msg="No ACME certificate generation required for domains [\"monitor.dev.***\"]." ACME CA="https://acme-staging-v02.api.letsencrypt.org/directory" providerName=letsEncrypt.acme routerName=traefik-secure@docker rule="Host(`monitor.dev.***`)"
traefik    | time="2023-04-29T10:22:05Z" level=debug msg="Serving default certificate for request: \"\""

You have two Traefik instances behind each other? Please explain your setup again.

Yes, I have a physical server on which proxmox (virtualization platform) is installed. In proxmox I am running virtual debian where docker is installed and traefik 1 is running on it. This traefik routes all requests from the *.dev.*** domain to IP 192.168.10.100 which is another VM with debian on which docker is installed with traefik 2 and other containers.
Here is the configuration of traefik 1.

docker-compose.yml

services:
  traefik:
    image: traefik:latest
    container_name: traefik
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    networks:
      - proxy
    ports:
      - 80:80
      - 443:443
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /root/traefik/data/traefik.yml:/traefik.yml:ro
      - /root/traefik/data/acme.json:/acme.json
      - /root/traefik/config.yml:/config.yml:ro
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik.entrypoints=http"
      - "traefik.http.routers.traefik.rule=Host(`monitor.***`)"
      - "traefik.http.middlewares.sslheader.headers.customrequestheaders.X-Forwarded-Proto=https"
      - "traefik.http.routers.traefik-secure.entrypoints=https"
      - "traefik.http.routers.traefik-secure.rule=Host(`monitor.***`)"
      - "traefik.http.routers.traefik-secure.tls=true"
      - "traefik.http.routers.traefik-secure.tls.certresolver=letsencrypt"
      - "traefik.http.routers.traefik-secure.service=api@internal"

networks:
  proxy:
    external: true

traefik.yml

api:
  dashboard: true
  debug: true
entryPoints:
  http:
    address: ":80"
  https:
    address: ":443"
serversTransport:
  insecureSkipVerify: true
providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false
  file:
    filename: /config.yml
certificatesResolvers:
  letsencrypt:
    acme:
      email: lukas.zatloukal@foxily.cz
      storage: acme.json
      tlsChallenge: {}
      caServer: https://acme-staging-v02.api.letsencrypt.org/directory

config.yml

http:
  routers:
    foo:
      entryPoints:
        - "https"
      rule: HostRegexp(`dev.***,`{subhost:[a-z]+}.dev.***`)
      middlewares:
        - default-headers
        #- https-redirectscheme
      service: foo
      tls: {}
    bar:
      entryPoints:
        - "http"
      rule: HostRegexp(`dev.***`,`{subhost:[a-z]+}.dev.***`)
      middlewares:
        - default-headers
      service: bar

  middlewares:
    https-redirectscheme:
      redirectScheme:
        scheme: https
        permanent: true
    default-headers:
      headers:
        frameDeny: true
        browserXssFilter: true
        contentTypeNosniff: true
        forceSTSHeader: true
        stsIncludeSubdomains: true
        stsPreload: true
        stsSeconds: 15552000
        customFrameOptionsValue: SAMEORIGIN
        customRequestHeaders:
          X-Forwarded-Proto: https

  services:
    foo:
      loadBalancer:
        servers:
          - url: "https://192.168.10.100:443/"
        passHostHeader: true
    bar:
      loadBalancer:
        servers:
          - url: "http://192.168.10.100:80/"
        passHostHeader: true

Where is config of Traefik 2? And of your app?

Have you thought about using Docker Swarm?

Traefik 2 configuration is above in the first post. Traefik 1 is the in second post.
Yes I've heard of Docker Swarm, but I don't know how to work with it. Is the solution I came up with nonsense?

It seems possible to use LE tlsChallenge on 1 and httpChallenge on 2. I assume you want the external and internal connection to be LE encrypted.

Certs are only automatically created when using Host() or HostSNI(). For wildcard certs you need dnsChallenge.

You could simply try setting all domains in rule=Host() || Host() || Host() on Traefik 1. Then use the dynamic configuration as you have it with loadbalancer.server.url to forward to Traefik 2.