How to Block AI Bots on Traefik: Complete 2026 Guide

Methods at a glance

1. robots.txt — serve from upstream

Traefik is a proxy — it doesn't serve static files. Serve robots.txt from your upstream application. For a dedicated static file, add a minimal nginx container to your Docker Compose stack and route /robots.txt requests to it:

# docker-compose.yml — robots.txt via nginx sidecar
services:
  traefik:
    image: traefik:v3
    command:
      - --providers.docker=true
      - --providers.docker.exposedbydefault=false
      - --entrypoints.web.address=:80
      - --entrypoints.websecure.address=:443
      - --certificatesresolvers.le.acme.tlschallenge=true
      - --certificatesresolvers.le.acme.email=admin@example.com
      - --certificatesresolvers.le.acme.storage=/letsencrypt/acme.json
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./letsencrypt:/letsencrypt

  # Serves robots.txt at /robots.txt
  robots:
    image: nginx:alpine
    volumes:
      - ./robots.txt:/usr/share/nginx/html/robots.txt:ro
      - ./nginx-robots.conf:/etc/nginx/conf.d/default.conf:ro
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.robots.rule=Host(`example.com`) && Path(`/robots.txt`)"
      - "traefik.http.routers.robots.entrypoints=websecure"
      - "traefik.http.routers.robots.tls.certresolver=le"
      - "traefik.http.services.robots.loadbalancer.server.port=80"

  app:
    image: myapp:latest
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.app.rule=Host(`example.com`)"
      - "traefik.http.routers.app.entrypoints=websecure"
      - "traefik.http.routers.app.tls.certresolver=le"
      - "traefik.http.services.app.loadbalancer.server.port=3000"

# nginx-robots.conf — minimal config for the robots sidecar
server {
    listen 80;
    location = /robots.txt {
        root /usr/share/nginx/html;
        access_log off;
    }
}

If your upstream app already serves /robots.txt (e.g. Next.js public/robots.txt), skip the sidecar entirely.

2. X-Robots-Tag — Headers middleware

Traefik's Headers middleware adds, removes, or modifies HTTP headers on requests and responses. customResponseHeaders injects headers into every response from the upstream.

Docker Compose labels:

services:
  app:
    image: myapp:latest
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.app.rule=Host(`example.com`)"
      - "traefik.http.routers.app.entrypoints=websecure"
      - "traefik.http.routers.app.tls.certresolver=le"
      - "traefik.http.routers.app.middlewares=security-headers@docker"
      - "traefik.http.services.app.loadbalancer.server.port=3000"

      # Headers middleware — adds X-Robots-Tag to all responses
      - "traefik.http.middlewares.security-headers.headers.customresponseheaders.X-Robots-Tag=noai, noimageai"
      - "traefik.http.middlewares.security-headers.headers.customresponseheaders.X-Content-Type-Options=nosniff"

Dynamic file config (dynamic.yml with file provider):

# dynamic.yml — loaded by Traefik file provider
http:
  middlewares:
    security-headers:
      headers:
        customResponseHeaders:
          X-Robots-Tag: "noai, noimageai"
          X-Content-Type-Options: "nosniff"
          X-Frame-Options: "SAMEORIGIN"

  routers:
    app:
      rule: "Host(`example.com`)"
      entryPoints:
        - websecure
      middlewares:
        - security-headers
      service: app-service
      tls:
        certResolver: le

  services:
    app-service:
      loadBalancer:
        servers:
          - url: "http://app:3000"

3. User-Agent blocking — plugin middleware

Traefik's plugin system (Traefik Plugin Catalog) lets you install community middleware plugins. The traefik-plugin-bot-blocker plugin blocks requests by User-Agent before they reach your upstream.

# traefik.yml (static config) — enable the plugin
experimental:
  plugins:
    bot-blocker:
      moduleName: "github.com/bots-garden/traefik-plugin-bot-blocker"
      version: "v0.1.0"

entryPoints:
  web:
    address: ":80"
  websecure:
    address: ":443"

providers:
  docker:
    exposedByDefault: false
  file:
    filename: /etc/traefik/dynamic.yml

# dynamic.yml — configure the plugin middleware
http:
  middlewares:
    block-ai-bots:
      plugin:
        bot-blocker:
          # List of User-Agent substrings to block
          botUserAgents:
            - "GPTBot"
            - "ChatGPT-User"
            - "ClaudeBot"
            - "Claude-Web"
            - "anthropic-ai"
            - "CCBot"
            - "Google-Extended"
            - "PerplexityBot"
            - "Amazonbot"
            - "Bytespider"
            - "YouBot"
            - "DuckAssistBot"
            - "meta-externalagent"
            - "MistralAI-Spider"
            - "oai-searchbot"
          blockedStatusCode: 403

  routers:
    app:
      rule: "Host(`example.com`)"
      entryPoints:
        - websecure
      middlewares:
        - block-ai-bots
        - security-headers
      service: app-service
      tls:
        certResolver: le

Plugin availability depends on the Traefik version and whether the plugin is in the official catalog. Check plugins.traefik.io for the current plugin list and versions.

4. Application-layer blocking

The most reliable approach with Traefik is to handle User-Agent blocking in your upstream service. Traefik passes the original User-Agent header through unchanged, so your app sees the real value.

// Next.js middleware.ts — works with any Traefik setup
import { NextResponse } from 'next/server';
import type { NextRequest } from 'next/server';

const BLOCKED_UA = /GPTBot|ChatGPT-User|ClaudeBot|Claude-Web|anthropic-ai|CCBot|Google-Extended|PerplexityBot|Amazonbot|Bytespider|YouBot|DuckAssistBot|meta-externalagent|MistralAI-Spider|oai-searchbot/i;

export function middleware(request: NextRequest) {
  const ua = request.headers.get('user-agent') ?? '';

  if (BLOCKED_UA.test(ua)) {
    return new NextResponse('Forbidden', { status: 403 });
  }

  return NextResponse.next();
}

export const config = {
  matcher: ['/((?!_next|robots.txt|favicon.ico).*)'],
};

# Express.js middleware (Node.js)
const BLOCKED_UA = /GPTBot|ChatGPT-User|ClaudeBot|anthropic-ai|CCBot|Google-Extended|PerplexityBot|Amazonbot|Bytespider/i;

app.use((req, res, next) => {
  if (req.path === '/robots.txt') return next(); // always allow
  if (BLOCKED_UA.test(req.get('user-agent') || '')) {
    return res.status(403).send('Forbidden');
  }
  next();
});

5. Kubernetes — IngressRoute + Middleware

In Kubernetes, Traefik uses Custom Resource Definitions (CRDs). Define a Middleware resource for the headers, then reference it in your IngressRoute.

# middleware.yaml — X-Robots-Tag via Traefik Middleware CRD
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  name: security-headers
  namespace: default
spec:
  headers:
    customResponseHeaders:
      X-Robots-Tag: "noai, noimageai"
      X-Content-Type-Options: "nosniff"
---
# ingressroute.yaml
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: app-ingress
  namespace: default
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`example.com`)
      kind: Rule
      middlewares:
        - name: security-headers    # references Middleware CRD above
      services:
        - name: app-service
          port: 3000
  tls:
    certResolver: le

Middleware CRDs are namespaced. To use a middleware from a different namespace, use the namespace@kubernetescrd reference format in the IngressRoute.

6. Complete traefik.yml (static config)

Static config defines entrypoints, providers, and certificate resolvers. It does not define routers, services, or middlewares — those live in dynamic config (Docker labels or file provider).

# traefik.yml (static config — requires restart to change)

api:
  dashboard: false   # Disable in production

entryPoints:
  web:
    address: ":80"
    http:
      redirections:
        entrypoint:
          to: websecure
          scheme: https
  websecure:
    address: ":443"
    http3: {}        # Enable HTTP/3

certificatesResolvers:
  le:
    acme:
      email: admin@example.com
      storage: /letsencrypt/acme.json
      tlsChallenge: {}

providers:
  docker:
    exposedByDefault: false
    network: proxy           # Only route services on this Docker network
  file:
    filename: /etc/traefik/dynamic.yml
    watch: true              # Hot-reload on file changes

log:
  level: INFO

accessLog:
  filePath: /var/log/traefik/access.log
  bufferingSize: 100

7. Full Docker Compose example

Complete Docker Compose stack: Traefik v3 with automatic HTTPS, security headers middleware, and an app service.

# docker-compose.yml
networks:
  proxy:
    external: true   # Create with: docker network create proxy

services:
  traefik:
    image: traefik:v3
    restart: unless-stopped
    networks:
      - proxy
    ports:
      - "80:80"
      - "443:443"
      - "443:443/udp"   # HTTP/3
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./traefik.yml:/etc/traefik/traefik.yml:ro
      - ./dynamic.yml:/etc/traefik/dynamic.yml:ro
      - ./letsencrypt:/letsencrypt
    labels:
      - "traefik.enable=true"

  app:
    image: myapp:latest
    restart: unless-stopped
    networks:
      - proxy
    # No ports exposed — Traefik handles all ingress
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=proxy"
      - "traefik.http.routers.app.rule=Host(`example.com`)"
      - "traefik.http.routers.app.entrypoints=websecure"
      - "traefik.http.routers.app.tls.certresolver=le"
      - "traefik.http.routers.app.middlewares=security-headers@file"
      - "traefik.http.services.app.loadbalancer.server.port=3000"

# dynamic.yml (watched by Traefik — no restart needed on changes)
http:
  middlewares:
    security-headers:
      headers:
        customResponseHeaders:
          X-Robots-Tag: "noai, noimageai"
          X-Frame-Options: "SAMEORIGIN"
          X-Content-Type-Options: "nosniff"
        # Force HTTPS in browser for 1 year:
        stsSeconds: 31536000
        stsIncludeSubdomains: true

Frequently asked questions