Traefik v2 in Swarm Mode and Portainer..Portainer node will not join portainer

Traefik Traefik v2
,
1

I used this guide to setup traefik:

I removed the consul configs to start with a more simple setup and try to build out.

I have then added a portainer stack and connected it to the traefik overlay network I have called proxy.

docker network create --driver=overlay proxy

Added the labels to the node, substituting node.id for the actual id

docker node update --label-add traefik=true "node.id"
docker node update --label-add portainer.portainer-data=true "node.id"

I have added and confirmed all of the correct DNS settings internal and external. However, portainer will simply not "join" and picked the traefik configurations. I have wokring on this for at least 5 hours and I am at my wits end. Hopefully someone will see what I am missing. All of my configs and some logs are below

traefik.host.yml

version: "3.7"
services:
  traefik:
    image: traefik:v2.1.2
    ports:
      - "80:80"
      - "443:443"
    configs:
      # Dynamic config
      - source: routers-config
        target: /conf.d/routers.toml
      - source: middlewares-config
        target: /conf.d/middlewares.toml
      - source: tls-config
        target: /conf.d/tls.toml

      # Static config
      - source: traefik-config
        target: /traefik.yml
    networks:
      - proxy
    volumes:
      - "./traefik-certificates:/letsencrypt"
      - "/var/run/docker.sock:/var/run/docker.sock"
    deploy:
      placement:
        constraints:
          - node.role == manager
          - node.labels.traefik == true
      update_config:
        # https://docs.docker.com/compose/compose-file/#update_config
        order: start-first

      labels:
        - "traefik.enable=true" # Enable Traefik, because we disabled expose a service by default

        - "traefik.http.routers.t.rule=Host(`traefik.domain.com`)" # Tell Traefik to create routre 't' and catch all requests with given Host
        - "traefik.http.routers.t.service=api@internal" # the router 't' will forward request to service api@internal

        - "traefik.http.routers.t.tls.certresolver=le" # the router 't' will use TLS certresolver called LE
        - "traefik.http.routers.t.entrypoints=websecure" # the router 't' should listen on both entrypoints

        #- "traefik.http.services.t.loadbalancer.server.port=8080" # the router 't' will balance incoming requests between servers listens on port 8080
        # - "traefik.http.services.t.loadbalancer.passhostheader=true"

        #- "traefik.http.routers.t.middlewares=authtraefik" # Tell Traefik, that for router 't' should use following middleware
        #- "traefik.http.middlewares.authtraefik.basicauth.users=${USERNAME?Variable USERNAME not set}:${HASHED_PASSWORD?Variable HASHED_PASSWORD not set}" #  Tell Traefik to creat middleware for the give name with following credntails (bcrypt)

        - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" # global redirect to https
        - "traefik.http.routers.http-catchall.entrypoints=web"
        - "traefik.http.routers.http-catchall.middlewares=redirect-to-https"
        - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"

        - "traefik.http.routers.ping.rule=Host(`traefik.domain.com`) && Path(`/ping`)"
        - "traefik.http.routers.ping.service=ping@internal"
        - "traefik.http.routers.ping.tls.certresolver=le"
        - "traefik.http.routers.ping.tls=true"

        # Dummy service for Docker Swarm
        - "traefik.http.services.dummy-service.loadbalancer.server.port=59999"
                 
networks:
  proxy:
    external: true

volumes:
  traefik-certificates:
  
configs:
  routers-config:
    name: routers-config
    file: ./conf.d/routers.toml
  middlewares-config:
    name: middlewares-config
    file: ./conf.d/middlewares.toml
  tls-config:
    name: tls-config
    file: ./conf.d/tls.toml
  traefik-config:
    name: traefik-config
    file: ./traefik.yml

# docker stack deploy -c traefik-host.yml traefik --prune
#https://medium.com/@jakub.hajek/container-orchestration-with-traefik-2-x-and-docker-swarm-with-canary-deployment-27e7ea62af6f



#docker stack deploy -c traefik-host.yml traefik --prune

traefik.yml

log:
  level: info
  format: json

accessLog:
  format: json
  bufferingSize: 5

api:
  dashboard: true
  debug: true
  insecure: true

serversTransport:
  insecureSkipVerify: true

providers:
  docker:
    watch: true
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false
    swarmMode: true
    network: proxy
    swarmModeRefreshSeconds: 5

  file:
    directory: /conf.d
    watch: true

entryPoints:
  web:
    address: ":80"
    forwardedHeaders:
      insecure: true

  websecure:
    address: ":443"
    forwardedHeaders:
      insecure: true

  ping:
    address: ":8082"
    
  metrics:
    address: ":8083"

certificatesResolvers:
  le:
    acme:
      email: rpelletier@domain.com
      storage: /letsencrypt/acme.json
      httpChallenge:
        entryPoint: http
      tlschallenge: true

ping:
  entryPoint: ping

metrics:
  prometheus:
    entryPoint: metrics

routers.toml

[http]
    [http.routers]
        [http.routers.redirecttohttps]
             entryPoints = ["web"]
             middlewares = ["https-redirect"]
             rule = "HostRegexp(`{host:.+}`)"
             service = "noop"

tls.toml

[tls]
  [tls.options]
    [tls.options.default]
#  https://community.containo.us/t/improving-the-ssl-rating/939/3
#    minVersion = "VersionTLS12"
      sniStrict = true
      cipherSuites = [
        "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
        "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", # TLS 1.2
        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", # TLS 1.2
        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
        "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
        "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
        "TLS_AES_128_GCM_SHA256",
        "TLS_AES_256_GCM_SHA384",
        "TLS_CHACHA20_POLY1305_SHA256",
        "TLS_FALLBACK_SCSV"
      ]

middlewares.toml

[http]
    [http.middlewares]
         [http.middlewares.https-redirect.redirectscheme]
              scheme = "https"
              permanent = true
         [http.middlewares.security-headers.headers]
              # CORS
              AccessControlAllowMethods = ["GET", "OPTIONS", "PUT"]
              AccessControlAllowOrigin = "origin-list-or-null"
              AccessControlMaxAge = 100
              #AddVaryHeader = true
              BrowserXssFilter = true
              ContentTypeNosniff = true
              ForceSTSHeader = true
              FrameDeny = true
              SSLRedirect = true
              STSIncludeSubdomains = true
              STSPreload = true
              ContentSecurityPolicy = "default-src 'self' 'unsafe-inline'"
              CustomFrameOptionsValue = "SAMEORIGIN"
              ReferrerPolicy = "same-origin"
              FeaturePolicy = "vibrate 'self'"
              STSSeconds = 315360000

portainer.yml


version: '3.3'

services:
  agent:
    image: portainer/agent
    environment:
      AGENT_CLUSTER_ADDR: tasks.agent
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/docker/volumes:/var/lib/docker/volumes
    networks:
      - agent-network
    deploy:
      mode: global
      placement:
        constraints:
          - node.platform.os == linux

  portainer:
    image: portainer/portainer
    command: -H tcp://tasks.agent:9001 --tlsskipverify
    volumes:
      - portainer-data:/data
    networks:
      - agent-network
      - proxy
    ports:
      - 9000:9000
    deploy:
      placement:
        constraints:
          - node.role == manager
          - node.labels.portainer.portainer-data == true
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.pt.rule=Host(`docker.domain.com`)"
      - "traefik.http.routers.pt.entrypoints=web"
      - "traefik.http.services.pt.loadbalancer.server.port=9000" # it seems you always need to give traefik a port so it 'notices' the service
      - "traefik.http.routers.pt-secured.rule=Host(`docker.domain.com`)"
      - "traefik.http.routers.pt-secured.entrypoints=websecure"
      - "traefik.http.routers.pt-secured.tls.certresolver=le"
      - "traefik.docker.network=proxy"
      - traefik.tags=proxy
networks:
  agent-network:
    attachable: true
  proxy:
    external:
      name: proxy

volumes:
  portainer-data:

traefik_agent log

      
time="2020-05-08T16:42:20Z" level=info msg="Configuration loaded from file: /traefik.yml"


{"level":"info","msg":"Traefik version 2.1.2 built on 2020-01-07T18:23:57Z","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://docs.traefik.io/v2.0/contributing/data-collection/\n","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Starting provider aggregator.ProviderAggregator {}","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Starting provider *file.Provider {\"directory\":\"/conf.d\",\"watch\":true}","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Starting provider *acme.Provider {\"email\":\"rpelletier@domain.com\",\"caServer\":\"https://acme-v02.api.letsencrypt.org/directory\",\"storage\":\"/letsencrypt/acme.json\",\"keyType\":\"RSA4096\",\"httpChallenge\":{\"entryPoint\":\"http\"},\"tlsChallenge\":{},\"ResolverName\":\"le\",\"store\":{},\"ChallengeStore\":{}}","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Testing certificate renew...","providerName":"le.acme","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Starting provider *docker.Provider {\"watch\":true,\"endpoint\":\"unix:///var/run/docker.sock\",\"defaultRule\":\"Host(`{{ normalize .Name }}`)\",\"swarmMode\":true,\"network\":\"proxy\",\"swarmModeRefreshSeconds\":5000000000}","time":"2020-05-08T16:42:20Z"}


{"level":"info","msg":"Starting provider *traefik.Provider {}","time":"2020-05-08T16:42:20Z"}
2

I should note that I am able to access traefik.domain.com without issue and see the dashboard.