Traefik routes to wrong port after restarting the container or after some random time

Traefik Traefik v3 (latest)
1

Hi everyone, we have a problem with routing. We're using the compose.yaml file below. The problem is:

  • Neo4j browser connects to db with port 7687. It works fine at first, requests are routed to that port.
  • After a while (randomly) or after restarting the image, all Neo4j browser requests are routed to the port 80, which is HTTP port.
  • Port 7687 keeps functioning, there's no problem with that, but Neo4j browser is not able to connect to the database now.

Correct behavior

reverse-proxy-1  |  2025-04-08T11:56:46Z DBG github.com/traefik/traefik/v3/pkg/server/service/loadbalancer/wrr/wrr.go:213 > Service selected by WRR: http://<SERVER_IP>:7687
reverse-proxy-1  |  <CLIENT_IP> - - [08/Apr/2025:11:56:46 +0000] "GET / HTTP/1.1" 0 0 "-" "-" 260 "neo4j-bolt-router@docker" "http://<SERVER_IP>:7687" 375ms

Faulty behavior

reverse-proxy-1  |  2025-04-08T12:36:47Z DBG github.com/traefik/traefik/v3/pkg/server/service/loadbalancer/wrr/wrr.go:213 > Service selected by WRR: http://<SERVER_IP>:80
reverse-proxy-1  |  <CLIENT_IP> - - [08/Apr/2025:12:36:47 +0000] "GET / HTTP/1.1" 200 4426 "-" "-" 31 "website-router@docker" "http://<SERVER_IP>:80" 6ms

services:

  website:
    image: ghcr.io/hubiodatalab/crossbar-website:main
    hostname: website

    labels:
      - "traefik.enable=true"

      # Listen to
      - "traefik.http.routers.website-router.rule=Host(`${MY_DOMAIN}`)"

      # Proxy pass to
      - "traefik.http.routers.website-router.service=website-service"
      - "traefik.http.services.website-service.loadbalancer.server.port=80"

      # TLS config
      - "traefik.http.routers.website-router.tls=true"
      - "traefik.http.routers.website-router.tls.certresolver=letsencrypt"

      - 'traefik.http.routers.website-router.middlewares=cors-neo4j'
      - 'traefik.http.middlewares.cors-neo4j.headers.customResponseHeaders.Access-Control-Allow-Origin=https://${MY_DOMAIN}/db https://${MY_DOMAIN}:7687'

      - "com.centurylinklabs.watchtower.enable=true"

    expose:
      - "80"
    restart: always

  neo4j:
    image: neo4j:5.26.0-community-bullseye
    hostname: neo4j
    expose:
      - "7474"
      - "7687"
        #- "443"
    environment:
      - NEO4J_server_default__listen__address=0.0.0.0
      - NEO4J_server_default__advertised__address=${MY_DOMAIN}
      
      - NEO4J_server_databases_default__to__read__only=true
      - NEO4J_PLUGINS=["apoc"]
      - NEO4J_dbms_usage__report_enabled=false
        
      - NEO4J_server_bolt_tls__level=OPTIONAL
      - NEO4J_server_bolt_enabled=true

      - NEO4J_dbms_ssl_policy_bolt_enabled=true
      - NEO4J_dbms_ssl_policy_bolt_base__directory=/var/lib/neo4j/certificates/bolt
      - NEO4J_dbms_ssl_policy_bolt_private__key=private.key
      - NEO4J_dbms_ssl_policy_bolt_public__certificate=public.crt
      - NEO4J_dbms_ssl_policy_bolt_trust__all=true

      - NEO4J_dbms_ssl_policy_https_base__directory=/var/lib/neo4j/certificates/https
      - NEO4J_dbms_ssl_policy_https_private__key=private.key
      - NEO4J_dbms_ssl_policy_https_public__certificate=public.crt

        # Memory config
      - NEO4J_server_memory_heap_max__size=6g
      - NEO4J_db_memory_transaction_max=2g

      - NEO4J_browser_post__connect__cmd=style https://crossbarv2.hubiodatalab.com/style.grass

        #- NEO4J_dbms_security_auth__enabled=false


    env_file:
      - path: ./neo4j-password.env
        required: true
    volumes:
      - /var/lib/neo4j/data:/var/lib/neo4j/data
      - neo4j-certificates:/var/lib/neo4j/certificates
    depends_on:
      - tls-dumper
    restart: always
    labels:
      - "traefik.enable=true"

      # Listen to
      - "traefik.http.routers.neo4j-router.rule=Host(`${MY_DOMAIN}`) && PathPrefix(`/db`)"

      # Proxy pass to
      - "traefik.http.routers.neo4j-router.service=neo4j-service"
      - "traefik.http.services.neo4j-service.loadbalancer.server.port=7474"

      # TLS config
      - "traefik.http.routers.neo4j-router.tls=true"
      - "traefik.http.routers.neo4j-router.tls.certresolver=letsencrypt"

      # Define a new middleware to strip the URL prefix before sending it to static-files
      - "traefik.http.middlewares.neo4j-proxypass.replacepathregex.regex=^/db/(.*)"
      - "traefik.http.middlewares.neo4j-proxypass.replacepathregex.replacement=/browser/$$1"
      - "traefik.http.middlewares.neo4j-add-slash.redirectregex.regex=(^.*/db$$)"
      - "traefik.http.middlewares.neo4j-add-slash.redirectregex.replacement=$$1/"

      # tell Traefik which middlewares we want to use on this container
      - "traefik.http.routers.neo4j-router.middlewares=neo4j-add-slash,neo4j-proxypass"

      # Neo4j browser websocket
      # Listen to
      - "traefik.http.routers.neo4j-bolt-router.rule=Host(`${MY_DOMAIN}`)" # && PathPrefix(`/db-bolt`)"

      - "traefik.http.routers.neo4j-bolt-router.entrypoints=bolt"

      # Proxy pass to
      - "traefik.http.routers.neo4j-bolt-router.service=neo4j-bolt-service"
      - "traefik.http.services.neo4j-bolt-service.loadbalancer.server.port=7687"

      # TLS config
      - "traefik.http.routers.neo4j-bolt-router.tls=true"
      # - "traefik.http.routers.neo4j-bolt-router.tls.passthrough=true"
      - "traefik.http.routers.neo4j-bolt-router.tls.certresolver=letsencrypt"

      - "traefik.http.middlewares.neo4j-sslheader.headers.customrequestheaders.X-Forwarded-Proto=https,wss"

      # Define a new middleware to strip the URL prefix before sending it to static-files
      - "traefik.http.middlewares.neo4j-bolt-proxypass.replacepathregex.regex=^/db-bolt/(.*)"
        #- "traefik.http.middlewares.neo4j-bolt-proxypass.replacepathregex.replacement=/browser/$$1"
        #- "traefik.http.middlewares.neo4j-add-slash.redirectregex.regex=(^.*/db$$)"
        #- "traefik.http.middlewares.neo4j-add-slash.redirectregex.replacement=$$1/"


      - "traefik.http.routers.neo4j-bolt-router.middlewares=neo4j-sslheader" #,neo4j-bolt-proxypass"

      - "com.centurylinklabs.watchtower.enable=true"

  reverse-proxy:
    image: traefik:v3.3.5
    command:
      - "--providers.docker"
      - "--providers.docker.exposedbydefault=false"

      # Logging
      - "--log.level=DEBUG"
      - "--accesslog=true"

      # TLS Certification
      - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true"
      - "--certificatesresolvers.letsencrypt.acme.email=hubiodatascilab@gmail.com"
      - "--certificatesresolvers.letsencrypt.acme.storage=${ACME_JSON_PATH}"
      - "--certificatesresolvers.letsencrypt.acme.caServer=https://acme-v02.api.letsencrypt.org/directory"
        #- "--certificatesresolvers.letsencrypt.acme.caServer=https://acme-staging-v02.api.letsencrypt.org/directory"

      # HTTP -> HTTPS
      - "--entrypoints.web.address=:80"
      - "--entrypoints.web.http.redirections.entrypoint.to=websecure"
      - "--entrypoints.web.http.redirections.entrypoint.scheme=https"

      - "--entryPoints.websecure.address=:443"
      - "--entryPoints.bolt.address=:7687"

      - "com.centurylinklabs.watchtower.enable=true"

    ports:
      - "80:80"
      - "443:443"
      - "7687:7687"
    volumes:
      # So that Traefik can listen to Docker events
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - letsencrypt:/letsencrypt

    restart: always


  tls-dumper:
    
    build: ./tls-dumper
    image: tls-dumper
    
    labels:
      traefik.enable: false
    
    environment:
      - ACME_JSON_PATH=${ACME_JSON_PATH}
      - MY_DOMAIN=${MY_DOMAIN}
    
    volumes:
      - letsencrypt:/letsencrypt
      - neo4j-certificates:/letsencrypt/neo4j-certificates
    
    depends_on:
      - reverse-proxy
    restart: always


  watchtower:
    image: containrrr/watchtower:1.7.1
    command:
      - "--label-enable"
      - "--rolling-restart"
      - "--interval"
      - "300"
      - "--include-restarting"
      - "--include-stopped"
      - "--revive-stopped"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

volumes:
  letsencrypt:
  neo4j-certificates: