Hello,
For weeks, I have been trying to solve this problem. The problem is that the X-Forwarded-For header does not contain the IP address of my client computer. Instead it shows 10.200.0.1 which is the gateway of the pod network of my (single node) Kubernetes K3s cluster.
I installed k3s using the following commands, which installed (and enabled) traefik (2.10) automatically. Operating system is Fedora 40, though that should not matter.
mkdir /home/k3s-local-storage-provisioner
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" sh -s - \
--cluster-cidr 10.200.0.0/16 \
--service-cidr 10.32.0.0/16 \
--default-local-storage-path /home/k3s-local-storage-provisioner \
--tls-san pserver1.dgh.witpass.co.uk \
--write-kubeconfig /root/k3s.conf
My IP setup is quite straight forward shown below:
[Home computer]
(46.46.192.119)
|
|
[Internet]
|
|
(92.207.195.116) - public Interface
[DSL Router] - DNAT rules for 22, 80, 443
(192.168.0.1) - LAN interface
|
|
[LAN] (192.168.0.0/24)
|
|
(192.168.0.241)
[K3s server]
(10.200.0.1)
|
|
[Pod Network](10.200.0.0/16)
|
|---Traefik-pod
|
|---whoami.dgh.witpass.co.uk
On the DSL router, i have ports 22, 80 and 443 forwarded (DNAT) to my server 192.168.0.241
The traefik pod has the following setup:
$ kubectl -n kube-system get pods,svc
NAME READY STATUS RESTARTS AGE
pod/local-path-provisioner-6c86858495-z8jmx 1/1 Running 0 4h10m
pod/coredns-6799fbcd5-jdl6r 1/1 Running 0 4h10m
pod/metrics-server-54fd9b65b-kt5kc 1/1 Running 0 4h10m
pod/helm-install-traefik-crd-grq29 0/1 Completed 0 4h10m
pod/svclb-traefik-1d9f9777-gpqmq 2/2 Running 0 4h9m
pod/traefik-56495f9946-dckjf 1/1 Running 0 29m
pod/helm-install-traefik-gzhpr 0/1 Completed 0 29m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-dns ClusterIP 10.32.0.10 <none> 53/UDP,53/TCP,9153/TCP 4h10m
service/metrics-server ClusterIP 10.32.121.78 <none> 443/TCP 4h10m
service/traefik LoadBalancer 10.32.146.17 192.168.0.241 80:32155/TCP,443:32573/TCP 4h9m
$ kubectl -n kube-system get pod traefik-56495f9946-dckjf -o yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
prometheus.io/path: /metrics
prometheus.io/port: "9100"
prometheus.io/scrape: "true"
creationTimestamp: "2024-05-10T16:00:53Z"
generateName: traefik-56495f9946-
labels:
app.kubernetes.io/instance: traefik-kube-system
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: traefik
helm.sh/chart: traefik-25.0.3_up25.0.0
pod-template-hash: 56495f9946
name: traefik-56495f9946-dckjf
namespace: kube-system
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: true
controller: true
kind: ReplicaSet
name: traefik-56495f9946
uid: a7189cc4-05f9-4b9c-a4fc-52e5d1a76b00
resourceVersion: "4943"
uid: 2a2ca52d-c747-464e-af61-986e725778b7
spec:
containers:
- args:
- --global.checknewversion
- --global.sendanonymoususage
- --entrypoints.metrics.address=:9100/tcp
- --entrypoints.traefik.address=:9000/tcp
- --entrypoints.web.address=:8000/tcp
- --entrypoints.websecure.address=:8443/tcp
- --api.dashboard=true
- --ping=true
- --metrics.prometheus=true
- --metrics.prometheus.entrypoint=metrics
- --providers.kubernetescrd
- --providers.kubernetesingress
- --providers.kubernetesingress.ingressendpoint.publishedservice=kube-system/traefik
- --entrypoints.websecure.http.tls=true
- --log.level=DEBUG
- --entryPoints.web.forwardedHeaders.insecure=true
- --entryPoints.websecure.forwardedHeaders.insecure=true
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: rancher/mirrored-library-traefik:2.10.7
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /ping
port: 9000
scheme: HTTP
initialDelaySeconds: 2
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 2
name: traefik
ports:
- containerPort: 9100
name: metrics
protocol: TCP
- containerPort: 9000
name: traefik
protocol: TCP
- containerPort: 8000
name: web
protocol: TCP
- containerPort: 8443
name: websecure
protocol: TCP
readinessProbe:
failureThreshold: 1
httpGet:
path: /ping
port: 9000
scheme: HTTP
initialDelaySeconds: 2
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 2
resources: {}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /data
name: data
- mountPath: /tmp
name: tmp
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-fbq78
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: pserver1.dgh.witpass.co.uk
preemptionPolicy: PreemptLowerPriority
priority: 2000000000
priorityClassName: system-cluster-critical
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 65532
runAsNonRoot: true
runAsUser: 65532
serviceAccount: traefik
serviceAccountName: traefik
terminationGracePeriodSeconds: 60
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- emptyDir: {}
name: data
- emptyDir: {}
name: tmp
- name: kube-api-access-fbq78
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2024-05-10T16:00:55Z"
status: "True"
type: PodReadyToStartContainers
- lastProbeTime: null
lastTransitionTime: "2024-05-10T16:00:53Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2024-05-10T16:00:56Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2024-05-10T16:00:56Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2024-05-10T16:00:53Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://e4578fe4a2d154c816ba5963f8345010d210e40ce2a06c1ae43825d62b8729bd
image: docker.io/rancher/mirrored-library-traefik:2.10.7
imageID: docker.io/rancher/mirrored-library-traefik@sha256:606c4c924d9edd6d028a010c8f173dceb34046ed64fabdbce9ff29b2cf2b3042
lastState: {}
name: traefik
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2024-05-10T16:00:54Z"
hostIP: 192.168.0.241
hostIPs:
- ip: 192.168.0.241
phase: Running
podIP: 10.200.0.15
podIPs:
- ip: 10.200.0.15
qosClass: BestEffort
startTime: "2024-05-10T16:00:53Z"
I added the CLI arguments myself using following file:
$ cat traefik-add-forwardedheaders-support.yaml
apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: traefik
namespace: kube-system
spec:
valuesContent: |-
additionalArguments:
- "--log.level=DEBUG"
- "--entryPoints.web.forwardedHeaders.insecure=true"
- "--entryPoints.websecure.forwardedHeaders.insecure=true"
The problem:
When I do curl whoami.dgh.witpass.co.uk from my computer at home, I get the following:
$ curl whoami.dgh.witpass.co.uk
Hostname: whoami-569544f6cf-zkk8p
IP: 127.0.0.1
IP: ::1
IP: 10.200.0.10
IP: fe80::1cf3:d2ff:fed8:33f9
RemoteAddr: 10.200.0.15:32896
GET / HTTP/1.1
Host: whoami.dgh.witpass.co.uk
User-Agent: curl/8.0.1
Accept: */*
Accept-Encoding: gzip
X-Forwarded-For: 10.200.0.1
X-Forwarded-Host: whoami.dgh.witpass.co.uk
X-Forwarded-Port: 80
X-Forwarded-Proto: http
X-Forwarded-Server: traefik-56495f9946-dckjf
X-Real-Ip: 10.200.0.1
I don't see the IP address of my client in the X-Forwarded-For and/or X-Real-Ip .
The IP of my home computer is:
$ curl -4 ifconfig.io
46.46.192.119
I searched the internet, and there are lots of complaints about this problem, with no clear solution. Some people suggest to use the following, without any explanation of why these IPs, and why in this order, etc.
- --entryPoints.web.forwardedHeaders.trustedIPs=127.0.0.1/32,10.42.0.0/16
I tried these too, (of-course -I adjusted IPs according to my setup), but no avail. Unfortunately the documentation on this is not clear, and I see lots of forums full of frustration.
I recently removed k3s from this server, and installed plain docker (and docker-compose) with Traefik 2.11, and it worked. Notice that I used bare minimum CLI arguments to start traefik. OS is still Fedora. (OS was not re-installed).
(same server)
$ cat /home/containers-runtime/traefik.pserver1.dgh.witpass.co.uk/docker-compose.server.yml
version: "3.3"
services:
traefik:
image: "traefik:v2.11"
restart: always
command: --api.insecure=true --providers.docker
ports:
- "80:80"
volumes:
- "/var/run/docker.sock:/var/run/docker.sock:ro"
networks:
- services-network
networks:
services-network:
external: true
$ cat /home/containers-runtime/whoami.dgh.witpass.co.uk/docker-compose.server.yml
version: "3.3"
services:
whoami.dgh.witpass.co.uk:
image: traefik/whoami
restart: always
labels:
- "traefik.enable=true"
- "traefik.http.routers.whoami.rule=Host(`whoami.dgh.witpass.co.uk`)"
- "traefik.http.services.whoami.loadbalancer.server.port=80"
networks:
- services-network
networks:
services-network:
external: true
Here is the evidence that i see correct IP (of my home computer) in the X-Forwarded-For header, when I access this from my home computer:
[kamran@kworkhorse ~]$ curl http://whoami.dgh.witpass.co.uk
Hostname: 99322c064894
IP: 127.0.0.1
IP: 172.18.0.2
RemoteAddr: 172.18.0.3:58494
GET / HTTP/1.1
Host: whoami.dgh.witpass.co.uk
User-Agent: curl/8.0.1
Accept: */*
Accept-Encoding: gzip
X-Forwarded-For: 84.215.13.63
X-Forwarded-Host: whoami.dgh.witpass.co.uk
X-Forwarded-Port: 80
X-Forwarded-Proto: http
X-Forwarded-Server: 1bb34336ccb0
X-Real-Ip: 84.215.13.63
[kamran@kworkhorse ~]$
It works on Traefik 1.7 too. Not writing details here, because it is the same, and it works.
Why doesn't it work with Traefik v2 on k3s?
Files used by the whoami deployment in k3s:
$ cat whoami-http-deployment-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: whoami
namespace: dev
labels:
app: whoami
spec:
replicas: 1
selector:
matchLabels:
app: whoami
template:
metadata:
labels:
app: whoami
spec:
containers:
- image: docker.io/traefik/whoami:v1.6.1
name: whoami
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: whoami
namespace: dev
spec:
ports:
- name: whoami
port: 80
targetPort: 80
selector:
app: whoami
$ cat whoami-http-ingress.yaml
kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
name: whoami
namespace: dev
annotations:
kubernetes.io/ingress.className: traefik
traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
rules:
# Ensure that the DNS address points to the public IP of this server/k3s cluster.
- host: whoami.dgh.witpass.co.uk
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: whoami
port:
number: 80
If applicable, please paste the log output in DEBUG level
$ kubectl -n kube-system logs -f traefik-56495f9946-dckjf
time="2024-05-10T16:00:54Z" level=info msg="Configuration loaded from flags."
time="2024-05-10T16:00:54Z" level=info msg="Traefik version 2.10.7 built on 2023-12-06T15:54:59Z"
time="2024-05-10T16:00:54Z" level=debug msg="Static configuration loaded {\"global\":{\"checkNewVersion\":true,\"sendAnonymousUsage\":true},\"serversTransport\":{\"maxIdleConnsPerHost\":200},\"entryPoints\":{\"metrics\":{\"address\":\":9100/tcp\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}},\"traefik\":{\"address\":\":9000/tcp\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}},\"web\":{\"address\":\":8000/tcp\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{\"insecure\":true},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}},\"websecure\":{\"address\":\":8443/tcp\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{\"insecure\":true},\"http\":{\"tls\":{}},\"http2\":{\"maxConcurrentStreams\":250},\"udp\":{\"timeout\":\"3s\"}}},\"providers\":{\"providersThrottleDuration\":\"2s\",\"kubernetesIngress\":{\"ingressEndpoint\":{\"publishedService\":\"kube-system/traefik\"}},\"kubernetesCRD\":{}},\"api\":{\"dashboard\":true},\"metrics\":{\"prometheus\":{\"buckets\":[0.1,0.3,1.2,5],\"addEntryPointsLabels\":true,\"addServicesLabels\":true,\"entryPoint\":\"metrics\"}},\"ping\":{\"entryPoint\":\"traefik\",\"terminatingStatusCode\":503},\"log\":{\"level\":\"DEBUG\",\"format\":\"common\"}}"
time="2024-05-10T16:00:54Z" level=info msg="Stats collection is enabled."
time="2024-05-10T16:00:54Z" level=info msg="Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration."
time="2024-05-10T16:00:54Z" level=info msg="Help us improve Traefik by leaving this feature on :)"
time="2024-05-10T16:00:54Z" level=info msg="More details on: https://doc.traefik.io/traefik/contributing/data-collection/"
time="2024-05-10T16:00:54Z" level=debug msg="Configured Prometheus metrics" metricsProviderName=prometheus
time="2024-05-10T16:00:54Z" level=info msg="Starting provider aggregator aggregator.ProviderAggregator"
time="2024-05-10T16:00:54Z" level=debug msg="Starting TCP Server" entryPointName=metrics
time="2024-05-10T16:00:54Z" level=debug msg="Starting TCP Server" entryPointName=web
time="2024-05-10T16:00:54Z" level=debug msg="Starting TCP Server" entryPointName=websecure
time="2024-05-10T16:00:54Z" level=debug msg="Starting TCP Server" entryPointName=traefik
time="2024-05-10T16:00:54Z" level=info msg="Starting provider *ingress.Provider"
time="2024-05-10T16:00:54Z" level=debug msg="*ingress.Provider provider configuration: {\"ingressEndpoint\":{\"publishedService\":\"kube-system/traefik\"}}"
time="2024-05-10T16:00:54Z" level=info msg="ingress label selector is: \"\"" providerName=kubernetes
time="2024-05-10T16:00:54Z" level=info msg="Creating in-cluster Provider client" providerName=kubernetes
time="2024-05-10T16:00:54Z" level=info msg="Starting provider *crd.Provider"
time="2024-05-10T16:00:54Z" level=debug msg="*crd.Provider provider configuration: {}"
time="2024-05-10T16:00:54Z" level=info msg="Starting provider *acme.ChallengeTLSALPN"
time="2024-05-10T16:00:54Z" level=debug msg="*acme.ChallengeTLSALPN provider configuration: {}"
time="2024-05-10T16:00:54Z" level=warning msg="CRDs API Group \"traefik.containo.us\" is deprecated, and its support will end starting with Traefik v3. Please use the API Group \"traefik.io\" instead." providerName=kubernetescrd
time="2024-05-10T16:00:54Z" level=warning msg="CRDs API Version \"traefik.io/v1alpha1\" will not be supported in Traefik v3 itself. However, an automatic migration path to the next version will be available." providerName=kubernetescrd
time="2024-05-10T16:00:54Z" level=info msg="label selector is: \"\"" providerName=kubernetescrd
time="2024-05-10T16:00:54Z" level=info msg="Creating in-cluster Provider client" providerName=kubernetescrd
time="2024-05-10T16:00:54Z" level=info msg="Starting provider *traefik.Provider"
time="2024-05-10T16:00:54Z" level=debug msg="*traefik.Provider provider configuration: {}"
time="2024-05-10T16:00:54Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"ping\":{\"entryPoints\":[\"traefik\"],\"service\":\"ping@internal\",\"rule\":\"PathPrefix(`/ping`)\",\"priority\":2147483647},\"prometheus\":{\"entryPoints\":[\"metrics\"],\"service\":\"prometheus@internal\",\"rule\":\"PathPrefix(`/metrics`)\",\"priority\":2147483647}},\"services\":{\"api\":{},\"dashboard\":{},\"noop\":{},\"ping\":{},\"prometheus\":{}},\"models\":{\"websecure\":{\"tls\":{}}},\"serversTransports\":{\"default\":{\"maxIdleConnsPerHost\":200}}},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=internal
time="2024-05-10T16:00:54Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"kube-system-traefik-dashboard-d012b7f875133eeab4e5\":{\"entryPoints\":[\"traefik\"],\"service\":\"api@internal\",\"rule\":\"PathPrefix(`/dashboard`) || PathPrefix(`/api`)\"}}},\"tcp\":{},\"udp\":{},\"tls\":{}}" providerName=kubernetescrd
time="2024-05-10T16:00:54Z" level=debug msg="Skipping ingress status update" namespace=dev ingress=whoami
time="2024-05-10T16:00:54Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"dev-whoami-pserver1-dgh-witlineinfra-tech\":{\"entryPoints\":[\"web\"],\"service\":\"dev-whoami-80\",\"rule\":\"Host(`pserver1.dgh.witpass.co.uk`) \\u0026\\u0026 PathPrefix(`/`)\"}},\"services\":{\"dev-whoami-80\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://10.200.0.10:80\"}],\"passHostHeader\":true}}}},\"tcp\":{},\"udp\":{}}" providerName=kubernetes
time="2024-05-10T16:00:54Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
time="2024-05-10T16:00:54Z" level=debug msg="Added outgoing tracing middleware ping@internal" routerName=ping@internal entryPointName=traefik middlewareName=tracing middlewareType=TracingForwarder
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" entryPointName=traefik middlewareName=traefik-internal-recovery middlewareType=Recovery
time="2024-05-10T16:00:54Z" level=debug msg="Added outgoing tracing middleware prometheus@internal" routerName=prometheus@internal middlewareName=tracing middlewareType=TracingForwarder entryPointName=metrics
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareName=traefik-internal-recovery middlewareType=Recovery entryPointName=metrics
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareType=Metrics entryPointName=web middlewareName=metrics-entrypoint
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareType=Metrics middlewareName=metrics-entrypoint entryPointName=websecure
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" entryPointName=metrics middlewareName=metrics-entrypoint middlewareType=Metrics
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" entryPointName=traefik middlewareName=metrics-entrypoint middlewareType=Metrics
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareType=Metrics entryPointName=web middlewareName=metrics-entrypoint
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" entryPointName=websecure middlewareName=metrics-entrypoint middlewareType=Metrics
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareType=Metrics entryPointName=metrics middlewareName=metrics-entrypoint
time="2024-05-10T16:00:54Z" level=debug msg="Creating middleware" middlewareType=Metrics entryPointName=traefik middlewareName=metrics-entrypoint
time="2024-05-10T16:00:55Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" routerName=dev-whoami-pserver1-dgh-witlineinfra-tech@kubernetes serviceName=dev-whoami-80 middlewareName=pipelining middlewareType=Pipelining entryPointName=web
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareType=Metrics middlewareName=metrics-service entryPointName=web routerName=dev-whoami-pserver1-dgh-witlineinfra-tech@kubernetes serviceName=dev-whoami-80
time="2024-05-10T16:00:55Z" level=debug msg="Creating load-balancer" serviceName=dev-whoami-80 entryPointName=web routerName=dev-whoami-pserver1-dgh-witlineinfra-tech@kubernetes
time="2024-05-10T16:00:55Z" level=debug msg="Creating server 0 http://10.200.0.10:80" entryPointName=web routerName=dev-whoami-pserver1-dgh-witlineinfra-tech@kubernetes serviceName=dev-whoami-80 serverName=0
time="2024-05-10T16:00:55Z" level=debug msg="child http://10.200.0.10:80 now UP"
time="2024-05-10T16:00:55Z" level=debug msg="Propagating new UP status"
time="2024-05-10T16:00:55Z" level=debug msg="Added outgoing tracing middleware dev-whoami-80" entryPointName=web routerName=dev-whoami-pserver1-dgh-witlineinfra-tech@kubernetes middlewareName=tracing middlewareType=TracingForwarder
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareName=traefik-internal-recovery middlewareType=Recovery entryPointName=web
time="2024-05-10T16:00:55Z" level=debug msg="Added outgoing tracing middleware ping@internal" entryPointName=traefik routerName=ping@internal middlewareName=tracing middlewareType=TracingForwarder
time="2024-05-10T16:00:55Z" level=debug msg="Added outgoing tracing middleware api@internal" entryPointName=traefik routerName=kube-system-traefik-dashboard-d012b7f875133eeab4e5@kubernetescrd middlewareName=tracing middlewareType=TracingForwarder
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" entryPointName=traefik middlewareName=traefik-internal-recovery middlewareType=Recovery
time="2024-05-10T16:00:55Z" level=debug msg="Added outgoing tracing middleware prometheus@internal" middlewareType=TracingForwarder middlewareName=tracing routerName=prometheus@internal entryPointName=metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" entryPointName=metrics middlewareName=traefik-internal-recovery middlewareType=Recovery
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" entryPointName=web middlewareName=metrics-entrypoint middlewareType=Metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" entryPointName=websecure middlewareName=metrics-entrypoint middlewareType=Metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareName=metrics-entrypoint middlewareType=Metrics entryPointName=metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareName=metrics-entrypoint entryPointName=traefik middlewareType=Metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareName=metrics-entrypoint middlewareType=Metrics entryPointName=web
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareType=Metrics entryPointName=websecure middlewareName=metrics-entrypoint
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareName=metrics-entrypoint middlewareType=Metrics entryPointName=metrics
time="2024-05-10T16:00:55Z" level=debug msg="Creating middleware" middlewareType=Metrics middlewareName=metrics-entrypoint entryPointName=traefik
time="2024-05-10T16:00:56Z" level=debug msg="Skipping ingress status update" namespace=dev ingress=whoami
time="2024-05-10T16:00:56Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints" providerName=kubernetes
time="2024-05-10T16:00:56Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints" providerName=kubernetescrd
time="2024-05-10T16:00:56Z" level=debug msg="Skipping ingress status update" ingress=whoami namespace=dev
time="2024-05-10T16:00:56Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints" providerName=kubernetes
time="2024-05-10T16:00:56Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints" providerName=kubernetescrd
time="2024-05-10T16:10:54Z" level=info msg="Anonymous stats sent to https://collect.traefik.io/9vxmmkcdmalbdi635d4jgc5p5rx0h7h8: {\"global\":{\"checkNewVersion\":true,\"sendAnonymousUsage\":true},\"serversTransport\":{\"maxIdleConnsPerHost\":200},\"entryPoints\":{\"metrics\":{\"address\":\"xxxx\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250}},\"traefik\":{\"address\":\"xxxx\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250}},\"web\":{\"address\":\"xxxx\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{\"insecure\":true},\"http\":{},\"http2\":{\"maxConcurrentStreams\":250}},\"websecure\":{\"address\":\"xxxx\",\"transport\":{\"lifeCycle\":{\"graceTimeOut\":\"10s\"},\"respondingTimeouts\":{\"idleTimeout\":\"3m0s\"}},\"forwardedHeaders\":{\"insecure\":true},\"http\":{\"tls\":{}},\"http2\":{\"maxConcurrentStreams\":250}}},\"providers\":{\"providersThrottleDuration\":\"2s\",\"kubernetesIngress\":{\"ingressEndpoint\":{\"publishedService\":\"xxxx\"}},\"kubernetesCRD\":{}},\"api\":{\"dashboard\":true},\"metrics\":{\"prometheus\":{\"buckets\":[0.1,0.3,1.2,5],\"addEntryPointsLabels\":true,\"addServicesLabels\":true,\"entryPoint\":\"metrics\"}},\"ping\":{\"entryPoint\":\"traefik\",\"terminatingStatusCode\":503},\"log\":{\"level\":\"DEBUG\",\"format\":\"common\"}}"
time="2024-05-10T16:10:54Z" level=warning msg="A new release has been found: 3.0.0. Please consider updating."