Custom rules.yml / Reach local 'machine' when using internal treafik network

Traefik Traefik v2 (latest)
1

I'm using an internal Traefik Network

version: '3.5'

services:
  traefik:
    image: traefik:2.2.7
    container_name: traefik
    restart: unless-stopped
    environment:
      CF_API_EMAIL: ""
      CF_API_KEY: "
    ports:
      - "81:80"
      - "443:443"
    networks:
      - web
    volumes:
      - ./conf/:/etc/traefik/
      - ./acme.json:/acme.json
      - /var/run/docker.sock:/var/run/docker.sock
      - traefik_data:/tmp

networks:
  web:
    external: true


volumes:
  traefik_data:

With this I can reach all my services on that machine, internet, and everything on the host just fine. Now I added a service via a KVM VM that I want to add, however the traefik container is unable to reach it (checked ping etc.)

http:
  routers:
    api:
      rule: Host(`###`)
      entrypoints:
        - web-secure
      service: api@internal
      middlewares:
        - auth
      tls:
        certResolver: le
    ha:
      entryPoints:
        - "websecure"
      rule: "Host(`###`)"
      service: ha-service
      tls:
        certResolver: le
    ha2:
      entryPoints:
        - "websecure"
      rule: "Host(`###`)"
      service: ha2-service
      tls:
        certResolver: le
    omv-chip:
      entryPoints:
        - "websecure"
      rule: "Host(`###`)"
      service: omv-chip-service
      tls:
        certResolver: le
    portainer:
      entryPoints:
        - "websecure"
      rule: "Host(`###`)"
      service: portainer-service
      tls:
        certResolver: le

    DefaultHTTPSRedirect:
      entryPoints:
        - "web"
      rule: "HostRegexp(`{host:.+}`)"
      middlewares:
        - sslredirect
      service: noop
      priority: 1

  services:
    ha-service:
      loadBalancer:
        servers:
          - url: http://192.168.178.22:8123
    ha2-service:
      loadBalancer:
        servers:
          - url: http://192.168.178.176:8123
    omv-chip-service:
      loadBalancer:
        servers:
          - url: http://192.168.178.22:80
    portainer-service:
      loadBalancer:
        servers:
          - url: http://192.168.178.22:9000
    noop:
      loadBalancer:
        servers:
          - url: http://localhost/

  middlewares:
    sslredirect:
      redirectScheme:
        scheme: https
      permanent: true
    auth:
      basicAuth:
        users:
          - '

It's not possible to use host networking together with a specific custom network. I also tried adding my MACVLAN network just for the sake of trying, but that broke my setup completly.

Any advice on how to achieve the connectivity to the local network (IP is all I need, I don't care for DNS)?

Greetings
David