502 Bad Gateway' caused by: dial tcp 172.30.0.8:29091: connect: connection refused"

docker-compose.yml first stack (central one, 1 per server):

version: '3.6'

networks:
  web:
    external: true
  internal:
    external:
      name: traefik-proxy

volumes:
   # Volume to store traefik certificates
   traefik-letsencrypt:

services:
# Skipping the other containers / central monitoring (1 / server)
  traefik:
    image: "traefik:v2.6"
    container_name: "traefik"
    command:
      - "--log.level=DEBUG"
      # http://ud2.REDACTED.com:8080/dashboard/#/
      - "--api.dashboard=true"
      - "--api.insecure=true"
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--providers.docker.endpoint=unix:///var/run/docker.sock"
      - "--providers.docker.network=traefik-proxy"
        # Redirect http to https
      - "--entrypoints.web.address=:80"
      - "--entrypoints.web.http.redirections.entryPoint.to=websecure"
      - "--entrypoints.web.http.redirections.entryPoint.scheme=https"
      - "--entrypoints.web.http.redirections.entrypoint.permanent=true"
        # Https : port 443, with letsencrypt certificates
      - "--entrypoints.websecure.address=:443"
      - "--certificatesresolvers.myresolver.acme.httpchallenge=true"
      - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web"
        # staging letsencrypt (no quotas, but self signed certificates), comment to move to prod
        #- "--certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory"
      - "--certificatesresolvers.myresolver.acme.email=postmaster@${DOMAIN}"
      - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
        # Expose prometheus metrics : KO - not working
        # logs :
        # level=error msg="entryPoint \"metrics\" doesn't exist" routerName=prometheus@internal entryPointName=metrics
        # level=error msg="no valid entryPoint for this router" routerName=prometheus@internal
      - "--metrics.prometheus=true"
      - '--metrics.prometheus.buckets=0.1,0.3,1.2,5.0'
      #- "--metrics.prometheus.entrypoint=metrics"
      #- "--accesslog=true"
        #- "--tracing=true"
    ports:
      - "80:80"
      - "443:443"
      - "8080:8080"
    volumes:
      - traefik-letsencrypt:/letsencrypt
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    networks:
            #- web
      - internal

Then I am running (potentially multiple times), thus the variables, and thus the other docker-compose.yml file since I instanciate it from different directories with different values setting the variables in different..env files :slightly_smiling_face:

version: "3.6"

networks:
  database: {}
  node: {}
  frontend:
    external:
      name: traefik-proxy

services:
#whoami works ok
  whoami:
    image: "traefik/whoami"
    container_name: "whoami"
    networks:
      - frontend
    labels:
      - traefik.enable=true
      - traefik.docker.network=traefik-proxy
      - traefik.http.routers.whoami.rule=Host(`whoami.ud2.REDACTED.com`)
      - traefik.http.routers.whoami.entrypoints=websecure
      - traefik.http.routers.whoami.tls.certresolver=myresolver
      - traefik.http.services.whoami.loadbalancer.server.port=80


#this one is KO (502 BAD Gateway) 
  mezos:
    image: ${mezos_docker_image}
    # entrypoint: ./mezos.exe
    command: run --chain-db="${DBURL}" --tezos-url=http://node:${node_port_rpc} --mezos-url="http://mezos:${mezos_port_rpc}" --more-logs --verbosity=debug
    env_file:
      ./.env
    environment:
      PROTO: ${MEZOS_PROTO:-11}
    depends_on:
      postgres:
        condition: service_healthy
      node:
        condition: service_healthy
      indexer-token-support:
        condition: service_started
      indexer-mempool:
        condition: service_started
    restart: always
    healthcheck:
      test: ["CMD", "wget", "--tries=1", "--spider", "http://mezos:${mezos_port_rpc}/version"]
      interval: 10s
      timeout: 5s
    hostname: ${specific}-mezos
    ports:
      - ${mezos_port_rpc}:${mezos_port_rpc}
    expose:
      - ${mezos_port_rpc}
    networks:
      - frontend
      - node
      - database
    labels:
      - traefik.enable=true
      - traefik.docker.network=traefik-proxy
      - traefik.http.routers.${specific_net}-mezos.rule=Host(`${specific_net}.ud2.REDACTED.com`)
      - traefik.http.routers.${specific_net}-mezos.entrypoints=websecure
      - traefik.http.routers.${specific_net}-mezos.tls.certresolver=myresolver
      - traefik.http.services.${specific_net}-mezos.loadbalancer.server.port=29091

Extended (redacted) traefik logs for the request resulting in 502 error

traefik         | 2022-03-17T01:29:09.911990360Z time="2022-03-17T01:29:09Z" level=debug msg="vulcand/oxy/roundrobin/rr: Forwarding this request to URL" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/version\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\",\"RawFragment\":\"\"},\"Proto\":\"HTTP/2.0\",\"ProtoMajor\":2,\"ProtoMinor\":0,\"Header\":{\"Accept\":[\"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\"],\"Accept-Encoding\":[\"gzip, deflate, br\"],\"Accept-Language\":[\"en-US,en;q=0.7,fr;q=0.3\"],\"Cache-Control\":[\"max-age=0\"],\"Cookie\":[\"ph_mqkwGT0JNFqO-zX2t0mW6Tec9yooaVu7xCBlXtHnt5Y_posthog=%7B%22distinct_id%22%3A%2217f760123c286c-055091e71de439-455a69-7e9000-17f760123c311b1%22%2C%22%24device_id%22%3A%2217f760123c286c-055091e71de439-455a69-7e9000-17f760123c311b1%22%7D\"],\"Dnt\":[\"1\"],\"Sec-Fetch-Dest\":[\"document\"],\"Sec-Fetch-Mode\":[\"navigate\"],\"Sec-Fetch-Site\":[\"none\"],\"Sec-Fetch-User\":[\"?1\"],\"Sec-Gpc\":[\"1\"],\"Te\":[\"trailers\"],\"Upgrade-Insecure-Requests\":[\"1\"],\"User-Agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\"],\"X-Forwarded-Host\":[\"specific.ud2.REDACTED.com\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"],\"X-Forwarded-Server\":[\"30e4173f9b1e\"],\"X-Real-Ip\":[\"<MY_IP>\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"specific.ud2.REDACTED.com\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<MY_IP>:61925\",\"RequestURI\":\"/version\",\"TLS\":null}" ForwardURL="http://172.30.0.8:29091"
traefik         | 2022-03-17T01:29:09.912135753Z time="2022-03-17T01:29:09Z" level=debug msg="'502 Bad Gateway' caused by: dial tcp 172.30.0.8:29091: connect: connection refused"
traefik         | 2022-03-17T01:29:09.912231383Z time="2022-03-17T01:29:09Z" level=debug msg="vulcand/oxy/roundrobin/rr: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/version\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\",\"RawFragment\":\"\"},\"Proto\":\"HTTP/2.0\",\"ProtoMajor\":2,\"ProtoMinor\":0,\"Header\":{\"Accept\":[\"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\"],\"Accept-Encoding\":[\"gzip, deflate, br\"],\"Accept-Language\":[\"en-US,en;q=0.7,fr;q=0.3\"],\"Cache-Control\":[\"max-age=0\"],\"Cookie\":[\"ph_mqkwGT0JNFqO-zX2t0mW6Tec9yooaVu7xCBlXtHnt5Y_posthog=%7B%22distinct_id%22%3A%2217f760123c286c-055091e71de439-455a69-7e9000-17f760123c311b1%22%2C%22%24device_id%22%3A%2217f760123c286c-055091e71de439-455a69-7e9000-17f760123c311b1%22%7D\"],\"Dnt\":[\"1\"],\"Sec-Fetch-Dest\":[\"document\"],\"Sec-Fetch-Mode\":[\"navigate\"],\"Sec-Fetch-Site\":[\"none\"],\"Sec-Fetch-User\":[\"?1\"],\"Sec-Gpc\":[\"1\"],\"Te\":[\"trailers\"],\"Upgrade-Insecure-Requests\":[\"1\"],\"User-Agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\"],\"X-Forwarded-Host\":[\"specific.ud2.REDACTED.com\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"],\"X-Forwarded-Server\":[\"30e4173f9b1e\"],\"X-Real-Ip\":[\"<MY_IP>\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"specific.ud2.REDACTED.com\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<MY_IP>:61925\",\"RequestURI\":\"/version\",\"TLS\":null}"

I've found this : 502 Bad Gateway connection refused error - #9 by jcisio but it didn't help :frowning:

Thanks in advance for any tips on how to fix / refactor all this !

Hello @copolycube,

This seems to be an error of communication between Traefik and the backend. This might be due to the port you defined for your mezos service.

1 Like

Hello @tommoulard thanks for your suggestion.
Would you have a suggestion on how I can do to check where the error resides ?

One of the strange things is that I am able to communicate fine with my mezos service on http://ud2.REDACTED.com:29091/version externally, which I want to avoid and force through traefik.

Edits :

$ sudo docker ps
CONTAINER ID   IMAGE                                                                             COMMAND                  CREATED          STATUS                    PORTS                                                                                                                 NAMES
44dc9fa11be3   traefik:v2.6                                                                      "/entrypoint.sh --lo…"   3 minutes ago    Up 3 minutes              0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   traefik
b245a64bc07c   registryxxx/mezos/mezos:v2.2.5                               "./run-mezos.bash ru…"   13 minutes ago   Up 13 minutes (healthy)   0.0.0.0:29091->29091/tcp, :::29091->29091/tcp                                                                         specific_mezos_1
[...]

$ docker exec -ti 44dc9fa11be3 sh
/ # apk add --no-cache curl
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/main/x86_64/APKINDEX.tar.gz
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/community/x86_64/APKINDEX.tar.gz
(1/4) Installing brotli-libs (1.0.9-r5)
(2/4) Installing nghttp2-libs (1.43.0-r0)
(3/4) Installing libcurl (7.79.1-r0)
(4/4) Installing curl (7.79.1-r0)
Executing busybox-1.33.1-r6.trigger
OK: 11 MiB in 20 packages
/ # curl -v http://specific_mezos_1:29091/version
*   Trying 172.30.0.8:29091...
* connect to 172.30.0.8 port 29091 failed: Connection refused
* Failed to connect to specific_mezos_1 port 29091 after 0 ms: Connection refused
* Closing connection 0
curl: (7) Failed to connect to specific_mezos_1 port 29091 after 0 ms: Connection refused
/ # curl -v http://specific-mezos:29091/version
*   Trying 172.30.0.8:29091...
* connect to 172.30.0.8 port 29091 failed: Connection refused
* Failed to connect to specific-mezos port 29091 after 0 ms: Connection refused
* Closing connection 0
curl: (7) Failed to connect to specific-mezos port 29091 after 0 ms: Connection refused
/ # ping specific-mezos
PING specific-mezos (172.30.0.8): 56 data bytes
64 bytes from 172.30.0.8: seq=0 ttl=64 time=0.050 ms
64 bytes from 172.30.0.8: seq=1 ttl=64 time=0.040 ms
64 bytes from 172.30.0.8: seq=2 ttl=64 time=0.038 ms
64 bytes from 172.30.0.8: seq=3 ttl=64 time=0.045 ms
^C
--- specific-mezos ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 0.038/0.043/0.050 ms

My understanding is that those open ports should allow traefik to connect to specific-mezos on those ports?

  • 0.0.0.0:29091->29091/tcp, :::29091->29091/tcp

Thanks in advance

When I explore my docker container specific-mezos, this is what I can see in the NetworkSettings, if this may help ?

        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "8bfb8fa79b43375dbd82354ff7d8f65d59c463ff9022e0a5611af1f5e145b277",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {
                "29091/tcp": [
                    {
                        "HostIp": "0.0.0.0",
                        "HostPort": "29091"
                    },
                    {
                        "HostIp": "::",
                        "HostPort": "29091"
                    }
                ]
            },
            "SandboxKey": "/var/run/docker/netns/8bfb8fa79b43",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "specific_database": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "dfacce4d6adc",
                        "mezos",
                        "specific-mezos"
                    ],
                    "NetworkID": "e4ab20f275f699983f6ac42abb70a1613060a5144862e57a2d892387524f49dd",
                    "EndpointID": "61892ee3b6bcfed422e1057bdbb69822d844103204774327211f4d505006c587",
                    "Gateway": "172.20.0.1",
                    "IPAddress": "172.20.0.2",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:ac:14:00:02",
                    "DriverOpts": null
                },
                "specific_node": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "dfacce4d6adc",
                        "mezos",
                        "specific-mezos"
                    ],
                    "NetworkID": "3c7216b90f29a606a72e288c0bbb914e56fd1f98931d39207bad3980295c8976",
                    "EndpointID": "8b391ff19b63a0d38f2c5db8c1027b40985b96198c3da48fe319907d8da9dc22",
                    "Gateway": "172.18.0.1",
                    "IPAddress": "172.18.0.2",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:ac:12:00:02",
                    "DriverOpts": null
                },
                "traefik-proxy": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "dfacce4d6adc",
                        "mezos",
                        "specific-mezos"
                    ],
                    "NetworkID": "833b67cf2d5df267bf2195ddc9d36abb950c78c97ab73dbeee3fb9f39baa87b8",
                    "EndpointID": "80872c00e61f411b372c7bc04368e787b42cbbed9a4e783a2fbb46801233da18",
                    "Gateway": "172.30.0.1",
                    "IPAddress": "172.30.0.8",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:ac:1e:00:08",
                    "DriverOpts": null
                }
            }

Workaround / Solution found by changing the option to my mezos service configured to accept
0.0.0.0:29091 port (i.e. accept connections from any network on 29091 port).

Thanks a lot @tommoulard for your comment, @dduportal for the previously mentioned post that helped me solve this and P.W, for the suggestion of using 0.0.0.0 on mezos.


Note :no_entry_sign:
I'm still wondering why there is no better networking / what is wrong on the networking level and why it was unable to establish properly communication by specifying the hostname relative to the shared network.

I tried adding container_name: and hostname:, but none of those options worked.

I guess it's probably more a docker issue than a traefik issue, though.

1 Like

This topic was automatically closed 3 days after the last reply. New replies are no longer allowed.