Docker-traefik-prometheus-grafana stack doesnt create traefik container upon service deploy

Traefik Traefik v2 (latest)
, , , , ,
1

Hey everyone,

I am trying to run a traefik-grafana-prometheus stack in my docker swarm node but for some reason my error logs are empty. I was able to grep and tail the docker service logs on my Ubuntu 20.04LTS and able to get some info.
I followed instructions from here for installing:


Any help would be greatly appreciated. :slight_smile:

docker-compose.yml

ronin@ninjaserver:~$ cat docker-traefik-prometheus/docker-compose.yml 
version: '3.7'
services:
  traefik:
    image: traefik:v2.3.2
    labels:
      - constraint.label=rproxy-public
      - traefik.enable=true
      - traefik.docker.network=rproxy-public
      - traefik.http.middlewares.admin-auth.basicauth.users=admin:$$apr1$$8rWpRRHx$$xSzTeoNckKYO/umQnKso91
      - traefik.http.routers.rproxy-public-https.rule=Host(`traefik.whiskeyonthe.rocks`)
      - traefik.http.routers.rproxy-public-https.entrypoints=web,web-secure
      - traefik.http.routers.rproxy-public-https.tls=true
      - traefik.http.routers.rproxy-public-https.service=api@internal
      - traefik.http.routers.rproxy-public-https.tls.certresolver=le
      - traefik.http.routers.rproxy-public-https.middlewares=admin-auth
      - traefik.http.routers.rproxy-public.tls.domains[0].main=whiskeyonthe.rocks
      - traefik.http.routers.rproxy-public.tls.domains[0].sans=*.whiskeyonthe.rocks
      - traefik.http.services.rproxy-public.loadbalancer.server.port=8080
    command:
      - --log.level=DEBUG
      - "--accesslog"
      - "--log"
      - "--api"
      - "--api.insecure=true"
      - "--metrics.prometheus=true"
      - "--metrics.prometheus.buckets=0.1,0.3,1.2,5.0"
      - "--providers.docker=true"
      - "--providers.docker.swarmmode=true"
      - "--providers.docker.watch"
      - "--providers.docker.exposedbydefault=false"
      - "--providers.docker.constraints=Label(`constraint-label`, `rproxy-public`)"
      #- "--pilot.token=undefined"
      - "--entrypoints.web.address=:80"
      - "--entrypoints.web.http.redirections.entryPoint.to=web-secure"
      - "--entrypoints.web.http.redirections.entrypoint.scheme=https"
      - "--entrypoints.web-secure.address=:443"
      - "--certificatesresolvers.le.acme.email=leonidas@whiskeyonthe.rocks"
      - "--certificatesresolvers.le.acme.storage=/letsencrypt/acme.json"
      - "--certificatesresolvers.le.acme.tlschallenge=true"
    extra_hosts:
      - EXTRA_HOSTS
    environment:
      - NODE_ID=63glxibxvxp4wvqpyrvnndvjt
      - EMAIL=leonidas@whiskeyonthe.rocks
      - DOMAIN=traefik.whiskeyonthe.rocks
      - USERNAME=admin
      - EXTRA_HOSTS="traefik.whiskeyonthe.rocks:185.193.127.125"
    networks:
      - rproxy-public
      - inbound
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - "/srv/traefik.whiskeyonthe.rocks/traefik/le:/letsencrypt"
      - traefik_data:/traefik
    ports:
      - target: 80
        published: 80
        mode: host
      - target: 443
        published: 443
        mode: host
      - target: 8080
        published: 8080
        protocol: tcp
        mode: ingress
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 512M
        reservations:
          cpus: '0.25'
          memory: 256M
      mode: global
      placement:
        constraints:
          - node.role == manager
          - node.labels.rproxy-public.traefik_data == true
      update_config:
        parallelism: 1
        delay: 10s
      restart_policy:
        condition: on-failure
  prometheus:
    image: prom/prometheus
    volumes:
      - /srv/prometheus.whiskeyonthe.rocks/prometheus:/etc/prometheus/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
      - '--web.console.templates=/usr/share/prometheus/consoles'
      
    networks:
      - inbound
    extra_hosts:
      - "prometheus.whiskeyonthe.rocks:185.193.127.125"
    deploy:
      labels:
       - "traefik.enable=true"
       - "traefik.http.routers.prometheus.rule=Host(`prometheus.whiskeyonthe.rocks`)"
       - "traefik.http.routers.prometheus.service=prometheus"
       - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
       - "traefik.docker.network=inbound"
       - "traefil.http.routers.prometheus.middlewares=auth"
       - "traefik.http.routers.prometheus.entrypoints=web,web-secure"
       - "traefik.http.routers.prometheus.tls.certresolver=le"
      placement:
        constraints:
        - node.role==manager
      restart_policy:
        condition: on-failure
  
  grafana:
    image: grafana/grafana
    depends_on:
      - prometheus
    volumes:
      - grafana_data:/var/lib/grafana
      - /srv/grafana.whiskeyonthe.rocks/grafana/provisioning/:/etc/grafana/provisioning/
    env_file:
      - /srv/grafana.whiskeyonthe.rocks/grafana/config.monitoring
    networks:
      - inbound
    extra_hosts: 
      - "grafana.whiskeyonth.rocks:185.193.127.125"
    user: "104"
    deploy:
      labels:
        - "traefik.http.routers.grafana.rule=Host(`grafana.whiskeyonthe.rocks`)"
        - "traefik.http.routers.grafana.service=grafana"
        - "traefik.http.services.grafana.loadbalancer.server.port=3000"
        - "traefik.docker.network=inbound"
        - "traefik.http.routers.grafana.middlewares=auth"
        - "traefik.http.routers.grafana.entrypoints=web,web-secure"
        - "traefik.http.routers.grafana.tls.certresolver=le"
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure

networks:
  rproxy-public:
    driver: overlay
    name: rproxy-public
  inbound:
    driver: overlay
    name: inbound

volumes:
    prometheus_data: {}
    grafana_data: {}
    traefik_data: {}

docker ps -a

ronin@ninjaserver:~/docker-traefik-prometheus$ docker ps -a
CONTAINER ID        IMAGE                    COMMAND                  CREATED              STATUS              PORTS                    NAMES
b6ea69c106d6        grafana/grafana:latest   "/run.sh"                About a minute ago   Up 56 seconds       3000/tcp                 traefik_grafana.1.nqc2rbsndlv8lobcdpoaetiyk
51fd3708c1fc        prom/prometheus:latest   "/bin/prometheus --cā€¦"   About a minute ago   Up 56 seconds       9090/tcp                 traefik_prometheus.1.2d2si57z2ur1vrmu1oec7nkuu
289bb8cfaa41        portainer/agent:latest   "./agent"                9 minutes ago        Up 9 minutes        0.0.0.0:9001->9001/tcp   portainer-agent_agent.63glxibxvxp4wvqpyrvnndvjt.nj8k0obxym4l0ouuma8xmvaer

list of docker services

ronin@ninjaserver:~$ docker service ls
ID                  NAME                    MODE                REPLICAS            IMAGE                    PORTS
iwh2qbl4lyb2        portainer-agent_agent   global              1/1                 portainer/agent:latest   
ow14d9gh2lzv        traefik_grafana         replicated          1/1                 grafana/grafana:latest   
z6lmvpzap7l3        traefik_prometheus      replicated          1/1                 prom/prometheus:latest   
lbx20uutet1z        traefik_traefik         global              0/0                 traefik:v2.3.2           *:8080->8080/tcp

docker daemon service error log

ronin@ninjaserver:~/docker-traefik-prometheus$ sudo journalctl -u docker.service | tail -n 50 | grep -i error
Nov 13 08:55:11 ninjaserver dockerd[127687]: time="2020-11-13T08:55:11.577532420Z" level=error msg="error reading the kernel parameter net.ipv4.vs.expire_nodest_conn" error="open /proc/sys/net/ipv4/vs/expire_nodest_conn: no such file or directory"

docker service logs show up blank

ronin@ninjaserver:~/docker-traefik-prometheus$ docker service logs -f --no-trunc traefik_traefik
2

I was able to get a seperate traefik instance working successfully and I was wondering what is the difference between the two that makes the initial YAML (config shown here) work and why the one above (docker-traefik-prometheus) doesn't work and is difficult to diagnose.

docker-compose.yml

ronin@ninjaserver:~/docker-traefik-letsencrypt$ cat docker-compose.yml
version: '3.7'

services:

  traefik:
    image: traefik:2.3.2
    ports:
      #- "80:80"
      #- "443:443"
      - target: 80
        published: 80
        mode: host
      - target: 443
        published: 443
        mode : host
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 512M
        reservations:
          cpus: '0.25'
          memory: 256M
      placement:
        constraints:
          # Make the traefik service run only on the node with this label
          # as the node with it has the volume for the certificates
          - node.labels.reverse-proxy.reverse-proxy-certificates == true
          - node.role == manager
      labels:
        - constraint-label=reverse-proxy
        - traefik.enable=true
        - traefik.docker.network=reverse-proxy
        #- traefik.http.middlewares.admin-auth.basicauth.users=${USERNAME}:${HASHED_PASSWORD}
        - traefik.http.middlewares.admin-auth.basicauth.users=admin:$$apr1$$8rWpRRHx$$xSzTeoNckKYO/umQnKso91
        - traefik.http.routers.reverse-proxy-https.rule=Host(`traefik.whiskeyonthe.rocks`)
        - traefik.http.routers.reverse-proxy-https.entrypoints=web,web-secure
        - traefik.http.routers.reverse-proxy-https.tls=true
        - traefik.http.routers.reverse-proxy-https.service=api@internal
        - traefik.http.routers.reverse-proxy-https.tls.certresolver=le
        - traefik.http.routers.reverse-proxy-https.middlewares=admin-auth
        - traefik.http.routers.reverse-proxy.tls.domains[0].main=whiskeyonthe.rocks
        - traefik.http.routers.reverse-proxy.tls.domains[0].sans=*.whiskeyonthe.rocks
        - traefik.http.services.reverse-proxy.loadbalancer.server.port=8080
    volumes:
      # Add Docker as a mounted volume, so that Traefik can read the labels of other services
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      # Mount the volume to store the certificates
      - "/srv/traefik.whiskeyonthe.rocks/certs:/certificates"
      - "/srv/traefik.whiskeyonthe.rocks/letsencrypt:/letsencrypt"
      #- "reverse-proxy-certificates:/certificates"
      #- "reverse-proxy-certificates:/letsencrypt"
    command:
      - --accesslog
      - '--pilot.token=71b1d833-0f8f-42e9-9087-3c117c81e327'
      - --log
      - --log.level=DEBUG
      - --api
      - --providers.docker
      - --providers.docker.constraints=Label(`constraint-label`, `reverse-proxy`)
      - --providers.docker.exposedbydefault=false
      - --providers.docker.swarmmode
      - --entrypoints.web.address=:80
      # global redirect to https
      - --entrypoints.web.http.redirections.entryPoint.to=web-secure
      - --entrypoints.web.http.redirections.entrypoint.scheme=https
      - --entrypoints.web-secure.address=:443
      - --certificatesresolvers.le.acme.email=${EMAIL}
      - --certificatesresolvers.le.acme.storage=/letsencrypt/acme.json
      - --certificatesresolvers.le.acme.tlschallenge=true
      #- --certificatesresolvers.le.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
    networks:
      # Use the public network created to be shared between Traefik and
      # any other service that needs to be publicly available with HTTPS
      - reverse-proxy
    extra_hosts:
      - EXTRA_HOSTS
    environment:
      - NODE_ID=63glxibxvxp4wvqpyrvnndvjt
      - EMAIL=leonidas@whiskeyonthe.rocks
      - DOMAIN=traefik.whiskeyonthe.rocks
      - USERNAME=admin
      - PASSWORD=12345
      #- USERPASS=echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g      
      #- $SRV_VPATH="/srv/DOMAIN.TLD/"
      - EXTRA_HOSTS="traefik.whiskeyonthe.rocks:185.193.127.125"
    env_file:
     - ./.env

  whoami:
    image: "traefik/whoami"
    container_name: "simple-service"
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.whoami.rule=Host(`whoami.whiskeyonthe.rocks`)"
      - "traefik.http.routers.whoami.middlewares=auth"
      - "traefik.http.routers.whoami.entrypoints=web,web-secure"
      - "traefik.http.routers.whoami.tls.certresolver=le"
    networks:
      - reverse-proxy

volumes:
  # Create a volume to store the certificates, there is a constraint to make sure
  # Traefik is always deployed to the same Docker node with the same volume containing
  # the HTTPS certificates
  reverse-proxy-certificates:
networks:
  # Use the previously created public network "reverse-proxy", shared with other
  # services that need to be publicly available via this Traefik
  reverse-proxy:
    external: true
    driver: overlay