Homeassistant External service 404 errors

Hello,

first time traefik user, swithcing from NPM. I've managed to get other containers on the same host as traefik proxied, both traefik and proxied services are placed in a docker network 'proxy', but i am struggling with HomeAssistant. My HomeAssistant runs as a separate kvm VM on the same docker host (OMV), so I expect that has something to do with it.

Checking the networking from the traefik container to my HA VM I noticed I'm not able to wget HA, but i can ping HA and also wget audiobookshelf using the docker host IP. FWIW i have disabled proxy options for cloudflare DNS resolutions

openmediavault:~/traefik# docker exec -it traefik /bin/sh
/ # ping 192.168.1.200
PING 192.168.1.200 (192.168.1.200): 56 data bytes
64 bytes from 192.168.1.200: seq=0 ttl=63 time=0.346 ms
64 bytes from 192.168.1.200: seq=1 ttl=63 time=0.428 ms
^C
--- 192.168.1.200 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.346/0.387/0.428 ms
!!!!try grabbing home assistant page!!!!!
/ # wget http://192.168.1.200:8123
Connecting to 192.168.1.200:8123 (192.168.1.200:8123)
wget: error getting response
/ #
!!!!Try grabbing Audiobookshelf page!!!!
# wget http://192.168.1.199:13378/
Connecting to 192.168.1.199:13378 (192.168.1.199:13378)
saving to 'index.html'
index.html           100% |***************************************************************************************************************************|  3879  0:00:00 ETA
'index.html' saved
/ #

traefik.yaml

api:
  dashboard: true
  debug: true
  insecure: true #enable API and dashboard on 8080 without needing a router

log:
  level: DEBUG

entryPoints:
  http:
    address: ":80"
    http:
      redirections:
        entryPoint:
          to: https
          scheme: https
  https:
    address: ":443"
serversTransport:
  insecureSkipVerify: true
providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false
  file:
#    directory: /root/traefik/conf/
    filename: /config.yaml  #File provider to configure external hosts and other manual settings
    watch: true
#    filename: /dynamic2.yaml
certificatesResolvers:
  cloudflare:
    acme:
      email: xxxxxxx@gmail.com #add your email
      storage: acme.json
      dnsChallenge:
        provider: cloudflare
        #disablePropagationCheck: true 
        resolvers:
          - "1.1.1.1:53"
          - "1.0.0.1:53"

config.yaml

    ha-router:
      service: ha-service
      rule: "Host(`ha.xxxxxxx.net`)"
      entryPoints:
        - http
      tls:
        certResolver: cloudflare
      middlewares:
        - default-headers
        - https-redirect
 #       - test-auth
  services:
    ha-service:
      loadBalancer:
        servers:
          - url: https://192.168.1.200:8123       #HTTPS Internal

  middlewares:
    https-redirect:
      redirectScheme:
        scheme: https
        permanent: true
#    test-auth:
#      basicAuth:
#        users:
#          - "xxxxxxxxxxxxxx"

    default-headers:
      headers:
        frameDeny: true
        sslRedirect: true
        browserXssFilter: true
        contentTypeNosniff: true
        forceSTSHeader: true
        stsIncludeSubdomains: true
        stsPreload: true
        stsSeconds: 15552000
        customFrameOptionsValue: SAMEORIGIN
        customRequestHeaders:
          X-Forwarded-Proto: https

    default-whitelist:
      ipWhiteList:
        sourceRange:
        - "10.0.0.0/8"
        - "192.168.0.0/16"
        - "172.16.0.0/12"
        - "100.64.0.0/10"

    secured:
      chain:
        middlewares:
        - default-headers

docker file

version: '3.5'

services:
  traefik:
    image: traefik:latest #2.10.0
    container_name: traefik
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    networks:
       proxy:
    ports:
      - 80:80
      - 443:443
      # The Web UI (enabled by --api.insecure=true)
      - 8081:8080
    environment:
      - CF_API_EMAIL=xxxxxxxxxxx@gmail.com
      - CF_DNS_API_TOKEN=xxxxxxxxxxxxxx
      # - CF_API_KEY=YOU_API_KEY
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /root/traefik/traefik.yaml:/traefik.yaml
      - /root/traefik/acme.json:/acme.json
      - /root/traefik/config.yaml:/config.yaml
      - /root/traefik/logs/traefik.log:/traefik.log
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik.entrypoints=http"
#      - "traefik.http.routers.traefik.rule=Host(`traefik-dashboard.xxxxxxxxxxxxx.net`)"
#      - "traefik.http.middlewares.traefik-auth.basicauth.users=xxxxxxxxxxxxxxxx"
#      - "traefik.http.middlewares.traefik-https-redirect.redirectscheme.scheme=https"
#      - "traefik.http.middlewares.sslheader.headers.customrequestheaders.X-Forwarded-Proto=https"
#      - "traefik.http.routers.traefik.middlewares=traefik-https-redirect"
      - "traefik.http.routers.traefik-secure.entrypoints=https"
#      - "traefik.http.routers.traefik-secure.rule=Host(`traefik-dashboard.xxxxxxxxxx.net`)"
#      - "traefik.http.routers.traefik-secure.middlewares=traefik-auth"
#      - "traefik.http.routers.traefik-secure.tls=true"
      - "traefik.http.routers.traefik-secure.tls.certresolver=cloudflare"
      - "traefik.http.routers.traefik-secure.tls.domains[0].main=xxxxxxxxx.net"
      - "traefik.http.routers.traefik-secure.tls.domains[0].sans=*.xxxxxxxxx.net"
      - "traefik.http.routers.traefik-secure.service=api@internal"

networks:
  proxy:
    name: proxy
    external: true

audiobookshelf docker file - Working example of a local container

services:
  audiobookshelf:
    image: ghcr.io/advplyr/audiobookshelf:latest
    container_name: audiobookshelf
#    network_mode: bridge
    networks:
      - proxy
    ports:
      - 13378:80
    environment:
      - PUID=1000
      - PGID=100
      - TZ=America/New_York
    volumes:
      - /srv/mergerfs/datapool1/data/media/audiobooks:/audiobooks
      - CHANGE_TO_COMPOSE_DATA_PATH/audiobookshelf:/config
      - CHANGE_TO_COMPOSE_DATA_PATH/audiobookshelf/metadata:/metadata
      #- /srv/dev-disk-by-uuid-811c2d2d-56c3-4fbd-a5d3-f643b0e6fc19/appdata/audiobookshelf:/config
      #- /srv/dev-disk-by-uuid-811c2d2d-56c3-4fbd-a5d3-f643b0e6fc19/appdata/audiobookshelf/metadata:/metadata
    restart: unless-stopped
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.audiobookshelf.entrypoints=http"
      - "traefik.http.routers.audiobookshelf.rule=Host(`abooks.xxxxxxxxxx.net`)"
#      - "traefik.http.middlewares.audiobookshelf-auth.basicauth.users=xxxxxxxxxxxx"
      - "traefik.http.middlewares.audiobookshelf-https-redirect.redirectscheme.scheme=https"
      - "traefik.http.middlewares.sslheader.headers.customrequestheaders.X-Forwarded-Proto=https"
      - "traefik.http.routers.audiobookshelf.middlewares=audiobookshelf-https-redirect"
      - "traefik.http.routers.audiobookshelf-secure.entrypoints=https"
      - "traefik.http.routers.audiobookshelf-secure.rule=Host(`abooks.xxxxxxx.net`)"
#      - "traefik.http.routers.audiobookshelf-secure.middlewares=audiobookshelf-auth"
      - "traefik.http.routers.audiobookshelf-secure.tls=true"
      - "traefik.http.routers.audiobookshelf-secure.tls.certresolver=cloudflare"

networks:
  proxy:
    external: true   #true = network created outside of this compose file.

docker inspect proxy network

openmediavault:~/traefik# docker inspect proxy
[
    {
        "Name": "proxy",
        "Id": "19c1ba703ab5f934f6d6c0e16ad28d374478877eea89ac43bbdd4d644a011b8c",
        "Created": "2024-09-23T21:13:46.939572878-04:00",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": {},
            "Config": [
                {
                    "Subnet": "172.19.0.0/16",
                    "Gateway": "172.19.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "9cccad4f3a84dac637668fc8dfdbba57eff7a4deff32ec0059f4317f33da2437": {
                "Name": "overseer",
                "EndpointID": "4f0dd4596b386313fecc0152d131261171859e024b014bfa387fb710cc52bb37",
                "MacAddress": "02:42:ac:13:00:04",
                "IPv4Address": "172.19.0.4/16",
                "IPv6Address": ""
            },
            "b2502a0451707bd37e1478d8f8d92eed6005319681103a7900f0106c68d2dfae": {
                "Name": "audiobookshelf",
                "EndpointID": "b864041695793a5e5caa2d709709fbce195b93ceccba24431114760206b0f2d0",
                "MacAddress": "02:42:ac:13:00:03",
                "IPv4Address": "172.19.0.3/16",
                "IPv6Address": ""
            },
            "d334f2763d6f31efb0acb35f8dfd23f2728af99a5e84e02673cf4a3a837e7965": {
                "Name": "traefik",
                "EndpointID": "a6dab985bf889db59aaba24e54f4fd65f969e9c8f4f56b426fb249042ac31e76",
                "MacAddress": "02:42:ac:13:00:02",
                "IPv4Address": "172.19.0.2/16",
                "IPv6Address": ""
            }
        },
        "Options": {},
        "Labels": {}
    }
]