How to Block AI Bots on Gleam + Wisp: Complete 2026 Guide

Step 1 — Bot detection module (src/ai_bots.gleam)

A Gleam const list — allocated once at module load. list.any short-circuits on first match. string.contains for substring check. The caller is responsible for lowercasing ua before passing it — keeps this module pure.

// src/ai_bots.gleam — bot detection module

import gleam/list
import gleam/string

const patterns = [
  // OpenAI
  "gptbot", "chatgpt-user", "oai-searchbot",
  // Anthropic
  "claudebot", "claude-web",
  // Common Crawl
  "ccbot",
  // Bytedance
  "bytespider",
  // Meta
  "meta-externalagent",
  // Perplexity
  "perplexitybot",
  // Google AI
  "google-extended", "googleother",
  // Cohere
  "cohere-ai",
  // Amazon
  "amazonbot",
  // Diffbot
  "diffbot",
  // AI2
  "ai2bot",
  // DeepSeek
  "deepseekbot",
  // Mistral
  "mistralai-user",
  // xAI
  "xai-bot",
  // You.com
  "youbot",
  // DuckDuckGo AI
  "duckassistbot",
]

/// Check if a User-Agent string belongs to a known AI bot.
/// ua should already be lowercased before calling.
pub fn is_ai_bot(ua: String) -> Bool {
  list.any(patterns, fn(pattern) { string.contains(ua, pattern) })
}

Step 2 — Middleware function (src/middleware.gleam)

The function signature is the Wisp middleware contract: fn(Request, fn(Request) -> Response) -> Response. Return directly for 403; call next(req) to continue. request.get_header returns Result(String, Nil) — the type system enforces the absent-header case.

// src/middleware.gleam — Wisp middleware via use keyword

import gleam/http/request
import gleam/result
import gleam/string
import wisp

import ai_bots

/// bot_blocker is a Wisp middleware function.
/// Signature: fn(Request, fn(Request) -> Response) -> Response
///
/// Call with: use req <- bot_blocker(req)
/// Gleam desugars use to: bot_blocker(req, fn(req) { ... })
pub fn bot_blocker(
  req: wisp.Request,
  next: fn(wisp.Request) -> wisp.Response,
) -> wisp.Response {
  // request.get_header returns Result(String, Nil)
  // result.unwrap provides "" as default if the header is absent.
  // HTTP headers are normalised to lowercase by gleam_http — use "user-agent".
  let ua =
    req
    |> request.get_header("user-agent")
    |> result.unwrap("")
    |> string.lowercase

  case ai_bots.is_ai_bot(ua) {
    True ->
      // Short-circuit: return 403 immediately.
      // The next callback is never called — inner handler never runs.
      wisp.response(403)
      |> wisp.set_header("X-Robots-Tag", "noai, noimageai")
      |> wisp.string_body("Forbidden")

    False ->
      // Pass to the next handler, then add X-Robots-Tag to the response.
      next(req)
      |> wisp.set_header("X-Robots-Tag", "noai, noimageai")
  }
}

Step 3 — Request router with use middleware

wisp.path_segments(req) returns the path as a List(String). Pattern match on it to route. Serve robots.txt at the top before applying the bot-blocker. All other paths go through use req <- middleware.bot_blocker(req).

// src/app.gleam — request router with bot-blocker middleware

import gleam/http
import gleam/http/request
import wisp

import middleware

const robots_txt = "User-agent: *
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: AmazonBot
Disallow: /

User-agent: Diffbot
Disallow: /
"

pub fn handle_request(req: wisp.Request) -> wisp.Response {
  // Serve robots.txt BEFORE the bot-blocker middleware.
  // All crawlers — including AI bots — must be able to read it.
  case wisp.path_segments(req) {
    ["robots.txt"] ->
      wisp.ok()
      |> wisp.set_header("Content-Type", "text/plain; charset=utf-8")
      |> wisp.string_body(robots_txt)

    _ -> {
      // Apply bot-blocker to all other routes.
      // use desugars to: middleware.bot_blocker(req, fn(req) { ... })
      use req <- middleware.bot_blocker(req)
      router(req)
    }
  }
}

fn router(req: wisp.Request) -> wisp.Response {
  case wisp.path_segments(req) {
    [] -> handle_home(req)
    ["health"] -> wisp.ok() |> wisp.string_body("ok")
    ["api", "data"] -> handle_api_data(req)
    _ -> wisp.not_found()
  }
}

fn handle_home(_req: wisp.Request) -> wisp.Response {
  let html = "<!DOCTYPE html>
<html>
<head>
  <meta name=\"robots\" content=\"noai, noimageai\">
  <title>My Site</title>
</head>
<body><h1>Welcome</h1></body>
</html>"
  wisp.ok()
  |> wisp.set_header("Content-Type", "text/html; charset=utf-8")
  |> wisp.string_body(html)
}

fn handle_api_data(req: wisp.Request) -> wisp.Response {
  use <- wisp.require_method(req, http.Get)
  wisp.ok()
  |> wisp.set_header("Content-Type", "application/json")
  |> wisp.string_body("{\"data\":\"protected\"}")
}

Step 4 — Start the server (src/main.gleam)

Wisp requires a secret_key_base for signed cookies and session encryption. wisp_mist.handler wraps your handler function for Mist (the underlying HTTP server).process.sleep_forever() keeps the BEAM process alive after startup.

// src/main.gleam — start Mist server with Wisp handler

import gleam/erlang/process
import mist
import wisp
import wisp_mist

import app

pub fn main() {
  wisp.configure_logger()
  let secret_key_base = wisp.random_string(64)

  let assert Ok(_) =
    wisp_mist.handler(app.handle_request, secret_key_base)
    |> mist.new
    |> mist.port(8080)
    |> mist.start_http

  process.sleep_forever()
}

// gleam.toml — dependencies
// [dependencies]
// gleam_stdlib = ">= 0.34.0, < 2.0.0"
// gleam_http = ">= 3.0.0, < 4.0.0"
// wisp = ">= 1.4.0, < 2.0.0"
// wisp_mist = ">= 1.4.0, < 2.0.0"
// mist = ">= 4.0.0, < 5.0.0"
// gleam_erlang = ">= 0.25.0, < 1.0.0"

Step 5 — Scoped bot-blocking and middleware stacking

Apply use only inside specific branches of your pattern match to scope the middleware. Multiple use lines in sequence compose naturally — each layer wraps the next. The order in code is the execution order.

// Scoped bot-blocking — only protect /api/* routes

pub fn handle_request(req: wisp.Request) -> wisp.Response {
  case wisp.path_segments(req) {
    // Public routes — no bot-blocker
    ["robots.txt"] -> serve_robots_txt()
    ["health"] -> wisp.ok() |> wisp.string_body("ok")
    [] -> handle_home(req)

    // Protected API routes — bot-blocker applied only here
    ["api", ..rest] -> {
      use req <- middleware.bot_blocker(req)
      handle_api(req, rest)
    }

    _ -> wisp.not_found()
  }
}

// Multiple middleware layers compose naturally with use
fn handle_api(req: wisp.Request, _path: List(String)) -> wisp.Response {
  // use desugars left-to-right — bot_blocker fires first, then auth_check
  use req <- middleware.rate_limiter(req)
  // inner handler
  wisp.ok()
  |> wisp.set_header("Content-Type", "application/json")
  |> wisp.string_body("{\"ok\":true}")
}

Gleam Wisp vs Elixir Plug vs Erlang Cowboy vs Phoenix