How to Block AI Bots on Litestar (Python): Complete 2026 Guide

Layer 1: robots.txt

Use StaticFilesConfig to serve /robots.txt. Static routes are registered at the router level — they resolve before any middleware dispatch:

# static/robots.txt

User-agent: *
Allow: /

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Amazonbot
User-agent: PerplexityBot
User-agent: YouBot
User-agent: Diffbot
User-agent: DeepSeekBot
User-agent: MistralBot
User-agent: xAI-Bot
User-agent: AI2Bot
Disallow: /

# app.py
from litestar import Litestar
from litestar.static_files import StaticFilesConfig

app = Litestar(
    static_files_config=[
        StaticFilesConfig(
            path='/robots.txt',
            directories=['./static'],
            html_mode=False,
        ),
        StaticFilesConfig(
            path='/static',
            directories=['./static'],
            html_mode=False,
        ),
    ],
    middleware=[...],
)

Layers 2, 3 & 4: AbstractMiddleware

Create middleware/ai_bot_blocker.py. Subclass AbstractMiddleware and override dispatch():

# middleware/ai_bot_blocker.py
from litestar.middleware import AbstractMiddleware
from litestar.types import ASGIApp, Receive, Scope, Send, Message
from litestar.response import Response
from litestar.status_codes import HTTP_403_FORBIDDEN

AI_BOTS = [
    'gptbot', 'chatgpt-user', 'claudebot', 'anthropic-ai',
    'ccbot', 'cohere-ai', 'bytespider', 'amazonbot',
    'applebot-extended', 'perplexitybot', 'youbot', 'diffbot',
    'google-extended', 'deepseekbot', 'mistralbot', 'xai-bot',
    'ai2bot', 'oai-searchbot', 'duckassistbot',
]

EXEMPT_PATHS = {'/robots.txt', '/sitemap.xml', '/favicon.ico'}


class AiBotBlocker(AbstractMiddleware):

    async def dispatch(self, scope: Scope, receive: Receive, send: Send) -> None:
        # Only handle HTTP requests (skip websocket, lifespan)
        if scope['type'] != 'http':
            await self.app(scope, receive, send)
            return

        path = scope.get('path', '')
        headers = dict(scope.get('headers', []))
        ua = headers.get(b'user-agent', b'').decode('latin-1', errors='replace').lower()

        # Set noai meta in scope state for templates
        if 'state' not in scope:
            scope['state'] = {}
        scope['state']['robots'] = 'noai, noimageai'

        # Exempt paths always pass through
        if path in EXEMPT_PATHS:
            await self.app(scope, receive, send)
            return

        # Block AI bots — respond without calling self.app
        if any(bot in ua for bot in AI_BOTS):
            response = Response(
                content='Forbidden: AI crawlers are not permitted.',
                status_code=HTTP_403_FORBIDDEN,
                media_type='text/plain',
            )
            await response(scope, receive, send)
            return

        # Pass through — wrap send to inject X-Robots-Tag
        await self.app(scope, receive, self._inject_header(send))

    @staticmethod
    def _inject_header(send: Send) -> Send:
        """Wrap send to add X-Robots-Tag on http.response.start."""
        async def patched_send(message: Message) -> None:
            if message['type'] == 'http.response.start':
                headers = list(message.get('headers', []))
                headers.append((b'x-robots-tag', b'noai, noimageai'))
                message = {**message, 'headers': headers}
            await send(message)
        return patched_send

Register in the app:

# app.py
from litestar import Litestar
from litestar.static_files import StaticFilesConfig
from middleware.ai_bot_blocker import AiBotBlocker

app = Litestar(
    route_handlers=[...],
    middleware=[AiBotBlocker],  # FIFO — first in list = outermost = runs first
    static_files_config=[
        StaticFilesConfig(path='/robots.txt', directories=['./static']),
    ],
)

# Run: uvicorn app:app --reload

DefineMiddleware — factory configuration

DefineMiddleware passes constructor arguments to your middleware class at registration time. Use it when you want configurable exemptions or bot lists:

# middleware/ai_bot_blocker.py — configurable version
class AiBotBlocker(AbstractMiddleware):

    def __init__(
        self,
        app: ASGIApp,
        bot_list: list[str] | None = None,
        exempt_paths: set[str] | None = None,
    ) -> None:
        super().__init__(app=app)
        self.bot_list = bot_list or AI_BOTS
        self.exempt_paths = exempt_paths or EXEMPT_PATHS

    async def dispatch(self, scope, receive, send) -> None:
        ...  # use self.bot_list, self.exempt_paths

# app.py
from litestar.middleware import DefineMiddleware
from middleware.ai_bot_blocker import AiBotBlocker, AI_BOTS, EXEMPT_PATHS

# Add extra exempt paths without modifying the class
bot_middleware = DefineMiddleware(
    AiBotBlocker,
    bot_list=AI_BOTS,
    exempt_paths=EXEMPT_PATHS | {'/api/health', '/api/ping'},
)

app = Litestar(
    middleware=[bot_middleware],
    ...
)

Route, Router & Controller-scoped middleware

Litestar supports middleware at every registration level. Middleware is inherited top-down — app → router → controller → route:

from litestar import Litestar, Router, Controller, get
from litestar.middleware import DefineMiddleware
from middleware.ai_bot_blocker import AiBotBlocker

bot_mw = DefineMiddleware(AiBotBlocker)

# 1. Router-scoped — all routes under /api get blocked
api_router = Router(
    path='/api',
    route_handlers=[...],
    middleware=[bot_mw],
)

# 2. Controller-scoped — all methods in this controller get blocked
class ApiController(Controller):
    path = '/api/v2'
    middleware = [bot_mw]

    @get('/items')
    async def list_items(self) -> list[str]:
        return ['item1', 'item2']

# 3. Route-scoped — only this one endpoint is blocked
@get('/sensitive-data', middleware=[bot_mw])
async def sensitive(self) -> dict:
    return {'data': '...'}

app = Litestar(
    route_handlers=[api_router, ApiController, sensitive],
    # App-level middleware would apply to everything
)

Middleware order — FIFO (not LIFO)

Litestar processes the middleware list left-to-right, with the first item as the outermost wrapper. This is FIFO — the opposite of Starlette's LIFO add_middleware():

app = Litestar(
    middleware=[
        AiBotBlocker,    # outermost — runs FIRST for requests, LAST for responses
        RateLimiter,     # inner
        RequestLogger,   # innermost — runs LAST for requests, FIRST for responses
    ]
)

# vs Starlette/FastAPI add_middleware() — LIFO:
# app.add_middleware(RequestLogger)  # outermost (added last = runs first)
# app.add_middleware(RateLimiter)
# app.add_middleware(AiBotBlocker)   # innermost (added first = runs last)

Litestar vs FastAPI vs Starlette — middleware comparison

class AiBotBlocker(AbstractMiddleware):
    async def dispatch(self, scope, receive, send):
        ua = dict(scope['headers']).get(b'user-agent', b'').decode().lower()
        if any(b in ua for b in AI_BOTS):
            await Response('Forbidden', status_code=403)(scope, receive, send)
            return
        await self.app(scope, receive, send)  # no call_next

class AiBotBlocker(BaseHTTPMiddleware):
    async def dispatch(self, request, call_next):
        ua = request.headers.get('user-agent', '').lower()
        if any(b in ua for b in AI_BOTS):
            return Response('Forbidden', status_code=403)
        response = await call_next(request)
        response.headers['X-Robots-Tag'] = 'noai, noimageai'
        return response

class AiBotBlocker:
    def __init__(self, app): self.app = app
    async def __call__(self, scope, receive, send):
        if scope['type'] == 'http':
            ua = dict(scope['headers']).get(b'user-agent', b'').decode().lower()
            if any(b in ua for b in AI_BOTS):
                await Response('Forbidden', status_code=403)(scope, receive, send)
                return
        await self.app(scope, receive, send)

Testing

Litestar ships with litestar.testing.TestClient (sync) and AsyncTestClient:

import pytest
from litestar.testing import TestClient
from app import app

@pytest.fixture
def client():
    with TestClient(app=app) as c:
        yield c

def test_blocks_ai_bot(client):
    response = client.get(
        '/articles/test',
        headers={'User-Agent': 'GPTBot/1.0'},
    )
    assert response.status_code == 403

def test_allows_browser(client):
    response = client.get(
        '/articles/test',
        headers={'User-Agent': 'Mozilla/5.0 (compatible)'},
    )
    assert response.status_code == 200
    assert response.headers.get('x-robots-tag') == 'noai, noimageai'

def test_robots_txt_accessible(client):
    # AI bots can always fetch robots.txt
    response = client.get(
        '/robots.txt',
        headers={'User-Agent': 'GPTBot/1.0'},
    )
    assert response.status_code == 200

AI bot User-Agent strings (2026)