Traefik is unable to forward request to backend server (elasticsearch)

Hi,

I am trying to configure traefik to send request to backend (elasticsearch).

Client --> Apache ---> Traefik ---> Elastic Search

I am firing up below curl call from my machine which will land on Apache --> Traefik (which works as expected)

$ curl -XGET 'https://dummy.yom.com:10000/_search/_cat/indices/dummyindex*'

However, traefik access log shows response 404 Not Found error from backend. I am wondering if its even reaching elasticsearch ( which is not running as service in docker swarm cluster )

{"ClientAddr":"10.0.22.105:35446","ClientHost":"10.0.22.105","ClientPort":"35446","ClientUsername":"-","DownstreamContentSize":19,"DownstreamStatus":404,"Duration":1518016,"OriginContentSize":19,"OriginDuration":31129,"OriginStatus":404,"Overhead":1486887,"RequestAddr":"dummy.yom.com:10000","RequestContentSize":0,"RequestCount":1,"RequestHost":"dummy.yom.com","RequestMethod":"GET","RequestPath":"/_cat/indices/dummy*","RequestPort":"11764","RequestProtocol":"HTTP/1.1","RequestScheme":"http","RetryAttempts":0,"StartLocal":"2021-07-21T15:36:56.572415331Z","StartUTC":"2021-07-21T15:36:56.572415331Z","entryPointName":"http","level":"info","msg":"","time":"2021-07-21T15:36:56Z"}

traefik.toml :

logLevel = "DEBUG"
[providers]
    [providers.docker]
    exposedByDefault = false
    watch = true
    swarmMode = false
    network = "${DOCKER_STACK}_network"

[log]
[accessLog]
    format = "json"

# Dynamic configuration
[http.routers]
    [http.routers.backend]
        entryPoints = ["http"]
        rule = "Host(`${ELASTICSEARCH}`) && PathPrefix (`/_search`)"
        service = "backend"
        priority = 1

[http.services]
    [http.services.backend]
        [http.services.backend.loadBalancer]
            [[http.services.backend.loadBalancer.servers]]
                url = "${ELASTICSEARCH}"

{"level":"info","msg":"Starting provider *acme.ChallengeTLSALPN {\"Timeout\":4000000000}","time":"2021-07-21T15:32:49Z"}
{"level":"debug","msg":"Configuration received from provider file: {\"http\":{\"routers\":{\"backend\":{\"entryPoints\":[\"http\"],\"service\":\"backend\",\"rule\":\"Host(`${ELASTICSEARCH}`) \\u0026\\u0026 PathPrefix (`/_search`)\",\"priority\":1}},\"services\":{\"backend\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://blahblah:9200\"}],\"passHostHeader\":true}}}},\"tcp\":{},\"udp\":{},\"tls\":{}}","providerName":"file","time":"2021-07-21T15:32:49Z"}
{"level":"debug","msg":"Configuration received from provider internal: {\"http\":{\"services\":{\"api\":{},\"dashboard\":{},\"noop\":{}},\"serversTransports\":{\"default\":{\"maxIdleConnsPerHost\":200}}},\"tcp\":{},\"tls\":{}}","providerName":"internal","time":"2021-07-21T15:32:49Z"}
{"entryPointName":"http","level":"debug","middlewareName":"pipelining","middlewareType":"Pipelining","msg":"Creating middleware","routerName":"backend@file","serviceName":"backend","time":"2021-07-21T15:32:49Z"}
{"entryPointName":"http","level":"debug","msg":"Creating load-balancer","routerName":"backend@file","serviceName":"backend","time":"2021-07-21T15:32:49Z"}
{"entryPointName":"http","level":"debug","msg":"Creating server 0 http://blahblah:9200","routerName":"backend@file","serverName":0,"serviceName":"backend","time":"2021-07-21T15:32:49Z"}
{"entryPointName":"http","level":"debug","middlewareName":"tracing","middlewareType":"TracingForwarder","msg":"Added outgoing tracing middleware backend","routerName":"backend@file","time":"2021-07-21T15:32:49Z"}
{"entryPointName":"http","level":"debug","middlewareName":"traefik-internal-recovery","middlewareType":"Recovery","msg":"Creating middleware","time":"2021-07-21T15:32:49Z"}
{"level":"debug","msg":"No default certificate, generating one","time":"2021-07-21T15:32:49Z"}
{"level":"debug","msg":"No default certificate, generating one","time":"2021-07-21T15:32:50Z"}

Please can you let me know what is missing here??

Your static and dynamic configurations are in the same file. They need to be separate.

https://doc.traefik.io/traefik/getting-started/configuration-overview/