Traefik V2 cannot find my socket-proxy

Traefik Traefik v2
1

Hey all. I had recently moved to new hardware and am in the process of re-creating my docker containers. Currently working on getting Traefik and Socket-Proxy running. I use a master compose file that includes all separate yml files for all other services I want to run. It's a bit clunky and maybe a bit overkill but it's the process I employ. If it helps I used [THIS] tutorial to get my last server up and running and it was working great for months.

SSL is NOT my issue.

All $VARIABLES are defined in a .env file. I have triple-checked that all of these are correct as well.

As it stands I am getting these log messages from Traefik:

time="2024-09-11T14:22:58-06:00" level=debug msg="FIXME: Got an status-code for which error does not match any expected type!!!" status_code=-1 error="Get \"http://socket-proxy:2375/v1.24/version\": dial tcp 192.168.91.254:2375: i/o timeout" module=api
time="2024-09-11T14:22:58-06:00" level=error msg="Failed to retrieve information of the docker client and server host: Get \"http://socket-proxy:2375/v1.24/version\": dial tcp 192.168.91.254:2375: i/o timeout" providerName=docker
time="2024-09-11T14:22:58-06:00" level=error msg="Provider connection error Get \"http://socket-proxy:2375/v1.24/version\": dial tcp 192.168.91.254:2375: i/o timeout, retrying in 296.457743ms" providerName=docker

Here's my master compose file: (removed non-related inclusions and secrets)

########################### NETWORKS ###########################
networks:
  default:
    driver: bridge
  socket_proxy:
    name: socket_proxy
    driver: bridge
    ipam:
      config:
        - subnet: 192.168.91.0/24
  t2_proxy:
    name: t2_proxy
    driver: bridge
    ipam:
      config:
        - subnet: 192.168.90.0/24

########################### SERVICES ###########################

  # CORE
  - compose/$HOSTNAME/socket-proxy.yml # DOCKER SECURITY NETWORK SOCKET
  - compose/$HOSTNAME/traefik.yml      # PROXY SERVER

traefik.yml:

services:
  # Traefik 2 - Reverse Proxy
  traefik:
    container_name: traefik
    image: traefik:2.10
    security_opt:
      - no-new-privileges:true
    restart: unless-stopped
    # profiles: ["core", "all"]
    networks:
      t2_proxy:
        ipv4_address: 192.168.90.254 # You can specify a static IP
      socket_proxy:
    command: # CLI arguments
      - --global.checkNewVersion=true
      - --global.sendAnonymousUsage=true
      - --entrypoints.web.address=:80
      - --entrypoints.websecure.address=:443
      - --entrypoints.traefik.address=:8080
      - --entrypoints.websecure.http.tls=true
      - --entrypoints.web.http.redirections.entrypoint.to=websecure
      - --entrypoints.web.http.redirections.entrypoint.scheme=https
      - --entrypoints.web.http.redirections.entrypoint.permanent=true
      - --api=true
      - --api.dashboard=true
      # - --api.insecure=true
      #- --serversTransport.insecureSkipVerify=true
      # Allow these IPs to set the X-Forwarded-* headers - Cloudflare IPs: https://www.cloudflare.com/ips/
      - --entrypoints.websecure.forwardedHeaders.trustedIPs=$CLOUDFLARE_IPS,$LOCAL_IPS
      - --log=true
      - --log.filePath=/logs/traefik.log
      - --log.level=DEBUG # (Default: error) DEBUG, INFO, WARN, ERROR, FATAL, PANIC
      - --accessLog=true
      - --accessLog.filePath=/logs/access.log
      - --accessLog.bufferingSize=100 # Configuring a buffer of 100 lines
      - --accessLog.filters.statusCodes=204-299,400-499,500-599
      - --providers.docker=true
      # - --providers.docker.endpoint=unix:///var/run/docker.sock # Disable for Socket Proxy. Enable otherwise.
      - --providers.docker.endpoint=tcp://socket-proxy:2375 # Enable for Socket Proxy. Disable otherwise.
      - --providers.docker.exposedByDefault=false
      - --providers.docker.network=t2_proxy 
      - --providers.docker.swarmMode=false
      - --entrypoints.websecure.http.tls.options=tls-opts@file
      # Add dns-cloudflare as default certresolver for all services. Also enables TLS and no need to specify on individual services
      - --entrypoints.websecure.http.tls.certresolver=dns-cloudflare
      - --entrypoints.websecure.http.tls.domains[0].main=$DOMAINNAME_1
      - --entrypoints.websecure.http.tls.domains[0].sans=*.$DOMAINNAME_1
      # - --entrypoints.websecure.http.tls.domains[1].main=$DOMAINNAME_2 # Pulls main cert for second domain
      # - --entrypoints.websecure.http.tls.domains[1].sans=*.$DOMAINNAME_2 # Pulls wildcard cert for second domain
      - --providers.file.directory=/rules # Load dynamic configuration from one or more .toml or .yml files in a directory
      - --providers.file.watch=true # Only works on top level files in the rules folder
      - --certificatesResolvers.dns-cloudflare.acme.caServer=https://acme-staging-v02.api.letsencrypt.org/directory # LetsEncrypt Staging Server - uncomment when testing
      - --certificatesResolvers.dns-cloudflare.acme.storage=/acme.json
      - --certificatesResolvers.dns-cloudflare.acme.dnsChallenge.provider=cloudflare
      - --certificatesResolvers.dns-cloudflare.acme.dnsChallenge.resolvers=1.1.1.1:53,1.0.0.1:53
      - --certificatesResolvers.dns-cloudflare.acme.dnsChallenge.delayBeforeCheck=90 # To delay DNS check and reduce LE hitrate
    ports:
      - target: 80
        published: 80
        protocol: tcp
        mode: host
      - target: 443
        published: 443
        protocol: tcp
        mode: host
      # - target: 8080 # need to enable --api.insecure=true
      #  published: 8085
      #  protocol: tcp
      #  mode: host
    volumes:
      - $DOCKERDIR/appdata/traefik2/rules/$HOSTNAME:/rules # Dynamic File Provider directory
      # - /var/run/docker.sock:/var/run/docker.sock:ro # Enable if not using Socket Proxy
      - $DOCKERDIR/appdata/traefik2/acme/acme.json:/acme.json # Certs File 
      - $DOCKERDIR/logs/$HOSTNAME/traefik:/logs # Traefik logs
    environment:
      - TZ=$TZ
      - CF_DNS_API_TOKEN_FILE=/run/secrets/cf_dns_api_token    
      - HTPASSWD_FILE=/run/secrets/basic_auth_credentials # HTTP Basic Auth Credentials
      - DOMAINNAME_1 # Passing the domain name to traefik container to be able to use the variable in rules. 
    secrets:
      - cf_dns_api_token
      - basic_auth_credentials
    labels:
      - "traefik.enable=true"
      # HTTP Routers
      - "traefik.http.routers.traefik-rtr.entrypoints=websecure"
      - "traefik.http.routers.traefik-rtr.rule=Host(`traefik.$DOMAINNAME_1`)"
      # Services - API
      - "traefik.http.routers.traefik-rtr.service=api@internal"
      # Middlewares
      - "traefik.http.routers.traefik-rtr.middlewares=middlewares-basic-auth@file" # For Basic HTTP Authentication

socket-proxy.yml:

services:
  # Docker Socket Proxy - Security Enchanced Proxy for Docker Socket
  socket-proxy:
    container_name: socket-proxy
    image: lscr.io/linuxserver/socket-proxy:latest
    security_opt:
      - no-new-privileges:true
    restart: unless-stopped
    # profiles: ["core", "all"] 
    networks:
      socket_proxy:
        ipv4_address: 192.168.91.254 # You can specify a static IP

    privileged: true # true for VM. false for unprivileged LXC container on Proxmox.
    ports:
      - "127.0.0.1:2375:2375" # Do not expose this to the internet with port forwarding
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
    environment:
      - LOG_LEVEL=info # debug,info,notice,warning,err,crit,alert,emerg
      ## Variables match the URL prefix (i.e. AUTH blocks access to /auth/* parts of the API, etc.).
      # 0 to revoke access.
      # 1 to grant access.
      ## Granted by Default
      - EVENTS=1
      - PING=1
      - VERSION=1
      ## Revoked by Default
      # Security critical
      - AUTH=0
      - SECRETS=0
      - POST=1 # Watchtower
      # Not always needed
      - BUILD=0
      - COMMIT=0
      - CONFIGS=0
      - CONTAINERS=1 # Traefik, Portainer, etc.
      - DISTRIBUTION=0
      - EXEC=0
      - IMAGES=1 # Portainer
      - INFO=1 # Portainer
      - NETWORKS=1 # Portainer
      - NODES=0
      - PLUGINS=0
      - SERVICES=1 # Portainer
      - SESSION=0
      - SWARM=0
      - SYSTEM=0
      - TASKS=1 # Portainer
      - VOLUMES=1 # Portainer

When I try to ping

socket-proxy:2375

from traefik - it finds the IP:

192.168.91.254

but cannot get any response. In my master you can see that I have the correct IPAM config subnet set to

192.168.91.0/24

So I am kind of at a loss here. Any information would be much appreciated. I am learning as I go and would love to learn - so if anyone has a better way to go about proxying the docker socket that works better with Traefik I would love to hear it. (Also I know that I am still using V2 - I'll start migrating once I can get over this headache! lol)

Thanks,
Zak

2

I am personally not a big fan of giving fixed IPs to containers. Docker DNS will handle everything for you. Try to ping (or wget) socket-proxy from within Traefik container.

3

I have also started to assign a fixed IP for Traefik. I had too many problems with it when I didn't do this.

Now to your problem. Within a network, in your case socket_proxy, you do not have to expose ports. If I see correctly, you limit the port to localhost with 127.0.0.1:2375:2375.

If you now want to access the port from another container. The container does not have localhost as its address, but in this case the container has the IP address of Traefik in the socket_proxy network. You have not statically assigned the IP address of Traefik within socket_proxy. That would be one way to solve the problem and accordingly set the entry from 127.0.0.1:2375:2375 to statictraefikip:2375:2375.

My preferred solution would be:
Since Traefik and your socket proxy are in the same network socket_proxy you can remove the ports item completely

services:
  # Docker Socket Proxy - Security Enchanced Proxy for Docker Socket
  socket-proxy:
    container_name: socket-proxy
    image: lscr.io/linuxserver/socket-proxy:latest
    security_opt:
      - no-new-privileges:true
    restart: unless-stopped
    # profiles: ["core", "all"] 
    networks:
      socket_proxy:
        ipv4_address: 192.168.91.254 # You can specify a static IP
    privileged: true # true for VM. false for unprivileged LXC container on Proxmox.
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
    environment:
      - LOG_LEVEL=info # debug,info,notice,warning,err,crit,alert,emerg
      ## Variables match the URL prefix (i.e. AUTH blocks access to /auth/* parts of the API, etc.).
      # 0 to revoke access.
      # 1 to grant access.
      ## Granted by Default
      - EVENTS=1
      - PING=1
      - VERSION=1
      ## Revoked by Default
      # Security critical
      - AUTH=0
      - SECRETS=0
      - POST=1 # Watchtower
      # Not always needed
      - BUILD=0
      - COMMIT=0
      - CONFIGS=0
      - CONTAINERS=1 # Traefik, Portainer, etc.
      - DISTRIBUTION=0
      - EXEC=0
      - IMAGES=1 # Portainer
      - INFO=1 # Portainer
      - NETWORKS=1 # Portainer
      - NODES=0
      - PLUGINS=0
      - SERVICES=1 # Portainer
      - SESSION=0
      - SWARM=0
      - SYSTEM=0
      - TASKS=1 # Portainer
      - VOLUMES=1 # Portainer

This means that the port is not published externally.

You can also set the option internal: true for socket_proxy. Then socket_proxy is completely cut off from the outside

4

This works for me with Docker Swarm and docker stack deploy. I use a trusted base image and just manually create a little docker socket proxy configuration inline. Socket proxy uses no ports:, as all connections happen within a Docker network.

services:
  traefik:
    image: traefik:v3.1
    hostname: '{{.Node.Hostname}}'
    ports:
      - published: 80
        target: 80
        protocol: tcp
        mode: host
      - published: 443
        target: 443
        protocol: tcp
        mode: host
    networks:
      - dockersock
      - proxy
    volumes:
      - letsencrypt:/letsencrypt
      #- /var/log:/var/log
    command:
      - --api.dashboard=true
      - --log.level=INFO
      #- --log.filepath=/var/log/traefik.log
      - --accesslog=true
      - --accesslog.format=json
      #- --accesslog.filepath=/var/log/traefik-access.log
      - --providers.swarm.network=proxy
      - --providers.swarm.endpoint=tcp://dockersock:2375
      - --providers.swarm.exposedByDefault=false
      - --entrypoints.web.address=:80
      - --entrypoints.web.http.redirections.entrypoint.to=websecure
      - --entryPoints.web.http.redirections.entrypoint.scheme=https
      - --entrypoints.websecure.address=:443
      - --entrypoints.websecure.asDefault=true
      - --entrypoints.websecure.http.tls.certresolver=myresolver
      - --certificatesresolvers.myresolver.acme.email=mail@example.com
      - --certificatesresolvers.myresolver.acme.tlschallenge=true
      - --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json
    deploy:
      mode: global
      placement:
        constraints:
          - node.role==manager
      labels:
        - traefik.enable=true
        - traefik.http.routers.mydashboard.rule=Host(`traefik.example.com`)
        - traefik.http.routers.mydashboard.service=api@internal
        - traefik.http.routers.mydashboard.middlewares=myauth
        - traefik.http.services.mydashboard.loadbalancer.server.port=1337
        - traefik.http.middlewares.myauth.basicauth.users=test:$$apr1$$H6uskkkW$$IgXLP6ewTrSuBkTrqE8wj/

  dockersock:
    image: nginx:alpine-slim
    hostname: '{{.Node.Hostname}}'
    #user: "0"
    #user: dockersocket:docker
    security_opt:
      - no-new-privileges:true
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
    cap_drop:
      - ALL
    networks:
      - dockersock
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    deploy:
      mode: global
      placement:
        constraints:
          - node.role==manager
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        ln -sf /dev/stdout /var/log/nginx/access.log
        ln -sf /dev/stderr /var/log/nginx/error.log

        #su - nginx

        cat <<'EOF' > /etc/nginx/nginx.conf
        user root;
        events { worker_connections  1024; }
        http {
            server {
                listen 2375;
                location ~ ^/(v1\.24/events|v1\.24/containers|v1\.24/services|v1\.24/version|v1\.24/networks|v1\.24/tasks) {
                    if ($$request_method != GET) { return 405; }
                    proxy_pass http://unix:/var/run/docker.sock;
                    proxy_set_header Host $$host;
                    proxy_set_header X-Real-IP $$remote_addr;
                    proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
                    proxy_set_header X-Forwarded-Proto $$scheme;
                }
                location / { return 405; }
            }
        }
        EOF

        echo ID
        id

        echo Starting nginx
        nginx -g "daemon off;"
    expose:
      - 2375

  whoami:
    image: traefik/whoami:v1.10
    hostname: '{{.Node.Hostname}}'
    networks:
      - proxy
    deploy:
      mode: global
      labels:
        - traefik.enable=true
        - traefik.http.routers.whoami.rule=Host(`whoami.example.com`) || PathPrefix(`/whoami`)
        - traefik.http.services.whoami.loadbalancer.server.port=80

networks:
  proxy:
    name: proxy
    driver: overlay
    attachable: true

  dockersock:
    name: dockersock
    driver: overlay
    attachable: true

volumes:
  letsencrypt:
    name: letsencrypt

For regular Docker you might need to adjust the script line:

location ~ ^/(v1\.24/events|v1\.24/containers|v1\.24/services|v1\.24/version|v1\.24/networks|v1\.24/tasks) {