Hi Everyone,
I'm new to traefik and I'm trying to configure it as a proxy on a nomad cluster. I'm able to get traefik working properly without a secure/tls entrypoint but as soon as I try to add a tls entrypoint I run into the following issues.
The first issue is I get the following errors (repeated) in stdout
2025-04-14T17:33:34Z ERR Loading configuration, retrying in 7.999285545s error="loading configuration: Unexpected response code: 403 (Permission denied)" providerName=nomad-dev
2025-04-14T17:33:35Z ERR Loading configuration, retrying in 2.990902199s error="loading configuration: Unexpected response code: 403 (Permission denied)" providerName=nomad-default
I've confirmed that this is not an issue with the client token for Nomad because, as I mentioned, everything works fine when tls isn't involved. I've also confirmed that my traefik.yml and tls.yml files have 644 permissions on the container.
Secondly it seems that traefik may not be loading/using the provided certificate. When I try to access one of the services traefik should be proxying for I can see that traefik is using it's default cert (not the provided self signed cert I was hoping for).
I hoped that this topic would resolve my issues but sadly it does not.
FWIW I've also tried starting this up in docker but run into the same errors.
Here are my configuration files:
Nomad Job file for Traefik
variable "TRAEFIK_CLIENT_TOKEN" {
type = string
}
job "traefik" {
datacenters = ["ifs"]
type = "service"
group "traefik" {
count = 1
network {
port "http"{
static = 80
}
port "https"{
static = 443
}
port "admin"{
static = 8080
}
}
service {
name = "traefik-https"
provider = "nomad"
port = "https"
}
volume "tls-certs" {
type = "host"
source = "tls-certs" # Match the name from the client config
read_only = true
}
volume "traefik-config" {
type = "host"
source = "traefik-config" # Match the name from the client config
read_only = true
}
task "server" {
driver = "docker"
config {
image = "traefik:3"
ports = ["admin", "http", "https"]
args = [
"--api.dashboard=true",
"--api.insecure=true", ### For Test only, please do not use that in production
"--configFile=/etc/traefik/traefik.yml",
"--providers.nomad.endpoint.address=http://${attr.unique.network.ip-address}:4646", ### IP to your nomad server
"--log.maxbackups=5"
]
}
volume_mount {
volume = "tls-certs"
destination = "/etc/ssl/traefik"
propagation_mode = "private"
}
volume_mount {
volume = "traefik-config"
destination = "/etc/traefik"
read_only = true
}
env {
NOMAD_TOKEN = var.TRAEFIK_CLIENT_TOKEN
}
}
}
}
traefik.yml
entryPoints:
http:
address: ":80"
http:
redirections:
entryPoint:
to: "https"
scheme: "https"
permanent: true
https:
address: ":443"
http:
tls: {}
traefik:
address: ":8080"
providers:
file:
directory: "/etc/traefik/config"
watch: true
nomad:
endpoint:
address: "http://10.30.30.150:4646"
token: "${NOMAD_TOKEN}"
namespaces:
- default
- dev
api:
dashboard: true
insecure: true
log:
level: ERROR
/etc/trafik/config/tls.yml
tls:
certificates:
- certFile: "/etc/ssl/traefik/my-cert.crt"
keyFile: "/etc/ssl/traefik/my-cert.key"
options:
default:
clientAuth:
clientAuthType: RequireAndVerifyClientCert
caFiles:
- "/etc/ssl/traefik/my-cert-ca.crt"
The command I use to start the job on Nomad is:
nomad job run -namespace dev -var=TRAEFIK_CLIENT_TOKEN=${TRAEFIK_CLIENT_TOKEN} -verbose traefik.nomad.hcl
Any help is appreciated