Help to implement Docker Swarm TCP TLS Client Authentication

Hi,

I'm trying to implement mutual authentication on MQTTS TCP connection but the Traefik keeps letting pass all connections, I'm missing something.

Docker Traefik configuration:

version: "3.7"

services:
  traefik:
    image: traefik:v2.2
    command:
      - "--log.level=INFO"
      - "--api=true"
      - "--providers.docker.exposedbydefault=false"
      # enable swarm mode
      - "--providers.docker.endpoint=unix:///var/run/docker.sock"
      - "--providers.docker.swarmMode=true"
      - "--providers.docker.network=local-network"
      # dynamic configs
      - "--providers.file.directory=/etc/traefik/conf/"
      # entrypoints
      - "--entrypoints.web.address=:80"
      - "--entrypoints.websecure.address=:443"
      - "--entrypoints.mqtt-backend.address=:1883"
      - "--entrypoints.mqtts-backend.address=:8883"
      - "--entrypoints.mqtt-frontend.address=:1884"
      - "--entrypoints.mqtts-frontend.address=:8884"
      # acme
      - "--certificatesresolvers.myresolver.acme.httpchallenge=true"
      - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web"
      - "--certificatesresolvers.myresolver.acme.email=hidden.email@email.com"
      - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
    ports:
      - 80:80
      - 443:443
      - 1883:1883
      - 8883:8883
      - 1884:1884
      - 8884:8884
    volumes:
      - ./letsencrypt:/letsencrypt
      - ./certs:/certs:ro
      - ./keys:/keys:ro
      - ./configs/emqxbackend.toml:/etc/traefik/conf/emqxbackend.toml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - traefik-public
      - local-network
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.traefik-dashboard.entrypoints=web"
        - "traefik.http.routers.traefik-dashboard.rule=Host(`hidden.dashboard.domain`)"
        # - "traefik.http.routers.traefik-dashboard.tls.certresolver=myresolver"
        - "traefik.http.routers.traefik-dashboard.service=api@internal"
        - "traefik.http.routers.traefik-dashboard.middlewares=auth"
        - "traefik.http.middlewares.auth.basicauth.users=test:$$apr1$$H6uskkkW$$IgXLP6ewTrSuBkTrqE8wj/"
        # Dummy service for Swarm port detection. The port can be any valid integer value.
        - "traefik.http.services.dummy-svc.loadbalancer.server.port=9999"
        - "traefik.http.middlewares.testIPwhitelist.ipwhitelist.sourcerange=127.0.0.1/32, 192.168.8.135"
        # - "traefik.http.middlewares.testIPwhitelist.ipwhitelist.ipstrategy.depth=2"

networks:
  traefik-public:
    external: true
  local-network:
    external: true

/etc/traefik/conf/emqxbackend.toml:

[tls.options]
  [tls.options.emqxbackend]
    [tls.options.emqxbackend.clientAuth]
      caFiles = ["/certs/client.crt"]
      clientAuthType = "RequireAndVerifyClientCert"

Docker EMQX backend:

version: "3.7"

services:
  emqx-backend01:
    image: emqx-backend:1.0.0
    environment:
      - "EMQX_LOADED_PLUGINS=\"emqx_management,emqx_auth_http,emqx_recon,emqx_retainer,emqx_dashboard\""
      - "EMQX_NAME=emqx"
      - "EMQX_HOST=emqx.backend01"
      - "EMQX_CLUSTER__DISCOVERY=static"
      - "EMQX_CLUSTER__STATIC__SEEDS=emqx@emqx.backend01, emqx@emqx.backend02, emqx@emqx.backend03"
      - "EMQX_NODE__COOKIE=cookie"
    volumes:
      - emqx-backend01-logs:/opt/emqx/log
    deploy:
      # replicas: 1
      placement:
        constraints:
          - node.labels.worker == 1
      restart_policy:
        condition: none
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=local-network"
        # http with redirection
        - "traefik.http.routers.emqx-backend-dashboard.entrypoints=web"
        - "traefik.http.routers.emqx-backend-dashboard.rule=Host(`hidden.domain`)"
        - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https"
        - "traefik.http.routers.emqx-backend-dashboard.middlewares=redirect-middleware"
        # https
        - "traefik.http.routers.emqx-backend-dashboard-secure.rule=Host(`hidden.domain`)"
        - "traefik.http.routers.emqx-backend-dashboard-secure.entrypoints=websecure"
        - "traefik.http.routers.emqx-backend-dashboard-secure.tls=true"
        # - "traefik.http.routers.emqx-backend-dashboard-secure.tls.certresolver=myresolver"
        - "traefik.http.routers.emqx-backend-dashboard-secure.service=service-emqx-backend-dashboard"
        - "traefik.http.services.service-emqx-backend-dashboard.loadbalancer.server.port=18083"
        # mqtts
        - "traefik.tcp.routers.emqx-backend-mqtts.entrypoints=mqtts-backend"
        - "traefik.tcp.routers.emqx-backend-mqtts.rule=HostSNI(`domain`)"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls=true"
        # - "traefik.tcp.routers.emqx-backend-mqtts.tls.certresolver=myresolver"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls.options=emqxbackend@file"
        - "traefik.tcp.routers.emqx-backend-mqtts.service=service-mqtts-backend"
        - "traefik.tcp.services.service-mqtts-backend.loadbalancer.server.port=1883"
    networks:
      local-network:
        aliases:
        - emqx.backend01
        - emqx.backend

networks:
  local-network:
    external: true

volumes:
  emqx-backend01-logs:

docker logs traefik

time="2020-07-31T08:42:37Z" level=info msg="Configuration loaded from flags."
time="2020-07-31T08:42:37Z" level=info msg="Traefik version 2.2.8 built on 2020-07-28T15:46:03Z"
time="2020-07-31T08:42:37Z" level=info msg="\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://docs.traefik.io/contributing/data-collection/\n"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider aggregator.ProviderAggregator {}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *file.Provider {\"directory\":\"/etc/traefik/conf/\",\"watch\":true}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *acme.Provider {\"email\":\"hidden.email@email.com\",\"caServer\":\"https://acme-v02.api.letsencrypt.org/directory\",\"storage\":\"/letsencrypt/acme.json\",\"keyType\":\"RSA4096\",\"httpChallenge\":{\"entryPoint\":\"web\"},\"ResolverName\":\"myresolver\",\"store\":{},\"ChallengeStore\":{}}"
time="2020-07-31T08:42:37Z" level=info msg="Testing certificate renew..." providerName=myresolver.acme
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *docker.Provider {\"watch\":true,\"endpoint\":\"unix:///var/run/docker.sock\",\"defaultRule\":\"Host(`{{ normalize .Name }}`)\",\"swarmMode\":true,\"network\":\"local-network\",\"swarmModeRefreshSeconds\":15000000000}"
time="2020-07-31T08:42:37Z" level=info msg="Starting provider *traefik.Provider {}"
1 Like

Problem solved, the problem was outside the Traefik.

Does this allow scaling up the number of replicas on worker nodes when needed? For example,

`docker service scale emqx.backend=5`

Or does the configuration require 1 replica?

Also, why are there two networks: traefik-public and local-network? Are they both overlay networks for the swarm?

That was the initial idea, but to create multiple EMQX cluster nodes I need to duplicate the container and give another name. I need to do this way because I was only able to make the cluster discovery work with a static list in the docker swarm.

Example:

services:
  emqx-backend01:
    image: emqx-backend:1.0.0
    environment:
      - "EMQX_LOADED_PLUGINS=\"emqx_management,emqx_auth_http,emqx_recon,emqx_retainer,emqx_dashboard\""
      - "EMQX_NAME=emqx"
      - "EMQX_HOST=emqx.backend01"
      - "EMQX_CLUSTER__DISCOVERY=static"
      - "EMQX_CLUSTER__STATIC__SEEDS=emqx@emqx.backend01, emqx@emqx.backend02"
      - "EMQX_NODE__COOKIE=cookie"
    volumes:
      - emqx-backend01-logs:/opt/emqx/log
    deploy:
      # replicas: 1
      placement:
        constraints:
          - node.labels.worker == 1
      restart_policy:
        condition: none
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=local-network"
        # http with redirection
        - "traefik.http.routers.emqx-backend-dashboard.entrypoints=web"
        - "traefik.http.routers.emqx-backend-dashboard.rule=Host(`hidden.domain`)"
        - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https"
        - "traefik.http.routers.emqx-backend-dashboard.middlewares=redirect-middleware"
        # https
        - "traefik.http.routers.emqx-backend-dashboard-secure.rule=Host(`hidden.domain`)"
        - "traefik.http.routers.emqx-backend-dashboard-secure.entrypoints=websecure"
        - "traefik.http.routers.emqx-backend-dashboard-secure.tls=true"
        # - "traefik.http.routers.emqx-backend-dashboard-secure.tls.certresolver=myresolver"
        - "traefik.http.routers.emqx-backend-dashboard-secure.service=service-emqx-backend-dashboard"
        - "traefik.http.services.service-emqx-backend-dashboard.loadbalancer.server.port=18083"
        # mqtts
        - "traefik.tcp.routers.emqx-backend-mqtts.entrypoints=mqtts-backend"
        - "traefik.tcp.routers.emqx-backend-mqtts.rule=HostSNI(`domain`)"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls=true"
        # - "traefik.tcp.routers.emqx-backend-mqtts.tls.certresolver=myresolver"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls.options=emqxbackend@file"
        - "traefik.tcp.routers.emqx-backend-mqtts.service=service-mqtts-backend"
        - "traefik.tcp.services.service-mqtts-backend.loadbalancer.server.port=1883"
    networks:
      local-network:
        aliases:
        - emqx.backend01
        - emqx.backend

emqx-backend02:
    image: emqx-backend:1.0.0
    environment:
      - "EMQX_LOADED_PLUGINS=\"emqx_management,emqx_auth_http,emqx_recon,emqx_retainer,emqx_dashboard\""
      - "EMQX_NAME=emqx"
      - "EMQX_HOST=emqx.backend02"
      - "EMQX_CLUSTER__DISCOVERY=static"
      - "EMQX_CLUSTER__STATIC__SEEDS=emqx@emqx.backend01, emqx@emqx.backend02"
      - "EMQX_NODE__COOKIE=cookie"
    volumes:
      - emqx-backend02-logs:/opt/emqx/log
    deploy:
      # replicas: 1
      placement:
        constraints:
          - node.labels.worker == 1
      restart_policy:
        condition: none
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=local-network"
        # http with redirection
        - "traefik.http.routers.emqx-backend-dashboard.entrypoints=web"
        - "traefik.http.routers.emqx-backend-dashboard.rule=Host(`hidden.domain`)"
        - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https"
        - "traefik.http.routers.emqx-backend-dashboard.middlewares=redirect-middleware"
        # https
        - "traefik.http.routers.emqx-backend-dashboard-secure.rule=Host(`hidden.domain`)"
        - "traefik.http.routers.emqx-backend-dashboard-secure.entrypoints=websecure"
        - "traefik.http.routers.emqx-backend-dashboard-secure.tls=true"
        # - "traefik.http.routers.emqx-backend-dashboard-secure.tls.certresolver=myresolver"
        - "traefik.http.routers.emqx-backend-dashboard-secure.service=service-emqx-backend-dashboard"
        - "traefik.http.services.service-emqx-backend-dashboard.loadbalancer.server.port=18083"
        # mqtts
        - "traefik.tcp.routers.emqx-backend-mqtts.entrypoints=mqtts-backend"
        - "traefik.tcp.routers.emqx-backend-mqtts.rule=HostSNI(`domain`)"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls=true"
        # - "traefik.tcp.routers.emqx-backend-mqtts.tls.certresolver=myresolver"
        - "traefik.tcp.routers.emqx-backend-mqtts.tls.options=emqxbackend@file"
        - "traefik.tcp.routers.emqx-backend-mqtts.service=service-mqtts-backend"
        - "traefik.tcp.services.service-mqtts-backend.loadbalancer.server.port=1883"
    networks:
      local-network:
        aliases:
        - emqx.backend02
        - emqx.backend

networks:
  local-network:
    external: true

volumes:
  emqx-backend01-logs:

Also, why are there two networks: traefik-public and local-network ? Are they both overlay networks for the swarm?

Yes, are both overlay networks.

Thanks! Very helpful.