Enabling trace id's for Tempo/Loki

Traefik Traefik v2 (latest)
1

Hey all,

I've been trying to get tracing working across my Grafana stack.

I've got Zipkin traces enabled and they're being sent through to Tempo absolutely fine, however because they're not in the logs I can't see what they are.

I've switched the Traefik log to DEBUG, and I can see them in journald/on disk, but surely there's a way to get the trace ID's into the access log along with each request?

At the moment, my debug output looks like this:

{
   "Request" : "{\"Method\":\"POST\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/api/v2/spans\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\",\"RawFragment\":\"\"},\"Proto\":\"HTTP/2.0\",\"ProtoMajor\":2,\"ProtoMinor\":0,\"Header\":{\"Accept-Encoding\":[\"gzip\"],\"Content-Length\":[\"1693\"],\"Content-Type\":[\"application/json\"],\"User-Agent\":[\"Go-http-client/2.0\"],\"X-B3-Parentspanid\":[\"3ca9c1429b4db6b1\"],\"X-B3-Sampled\":[\"1\"],\"X-B3-Spanid\":[\"363b0911afaf9226\"],\"X-B3-Traceid\":[\"66e33a7b0478abff3ca9c1429b4db6b1\"],\"X-Forwarded-Host\":[\"otlpzipkin.service.consul\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"],\"X-Forwarded-Server\":[\"hypervisor01.servers.consul\"],\"X-Real-Ip\":[\"10.231.1.21\"]},\"ContentLength\":1693,\"TransferEncoding\":null,\"Host\":\"otlpzipkin.service.consul\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"x.x.x.x:51476\",\"RequestURI\":\"/api/v2/spans\",\"TLS\":null}",
   "level" : "debug",
   "msg" : "vulcand/oxy/roundrobin/rr: completed ServeHttp on request",
   "time" : "2021-11-30T10:13:21Z"
}

and my Traefik config is as follows:

[entrypoints]
  [entrypoints.http]
    address = ":80"
  [entrypoints.https]
    address = ":443"
    [entryPoints.https.http.tls]
      certResolver = "le"
      [[entryPoints.https.http.tls.domains]]
          main = "*.service.consul"

[http.serversTransports.mytransport]
  insecureSkipVerify = true

[providers]
  [providers.consulCatalog]
    prefix = "traefik"
    requireConsistent = true     
    exposedByDefault = true
    defaultRule = "Host(`{{ .Name }}.service.consul`)"
    [providers.consulCatalog.endpoint]
      address = "http://x.x.x.x:8500"
      scheme = "http"

[api]
  dashboard = true
  insecure = true
  debug = true

[metrics]
  [metrics.prometheus]
    addEntryPointsLabels = true
    addRoutersLabels = true
    addServicesLabels = true


[log]
  level = "DEBUG"
  filePath = "/var/log/traefik/traefik.log"
  format = "json"

[accessLog]
  filePath = "/var/log/traefik/access.log"
  format = "json"

[tracing]
  serviceName = "traefik"
  [tracing.zipkin]
    httpEndpoint = "https://otlpzipkin.service.consul/api/v2/spans"

# Enable ACME (Let's Encrypt): automatic SSL.
[certificatesResolvers.le.acme]
  email = "x@xxx.xxx"
  storage = "/etc/traefik.d/acme.json"
  [certificatesResolvers.le.acme.dnsChallenge]
    provider = "digitalocean"
    delayBeforeCheck = 10
    resolvers = ["1.1.1.1:53", "8.8.8.8:53"]

I'm sure I'm missing something simple, but I can't find it in the docs!