How to Block AI Bots on Elixir Phoenix: Complete 2026 Guide

Methods at a glance

1. robots.txt — Plug.Static

Place robots.txt in priv/static/. Phoenix's default endpoint includes Plug.Static which serves files from this directory at the root URL path. No code changes needed — just drop the file in.

priv/static/robots.txt

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: *
Allow: /

lib/my_app_web/endpoint.ex — verify Plug.Static config

# lib/my_app_web/endpoint.ex
# Default Phoenix endpoint already has this — just verify it includes robots.txt

plug Plug.Static,
  at: "/",
  from: :my_app,
  gzip: false,
  # If you use "only:", make sure robots.txt is listed:
  only: ~w(assets fonts images favicon.ico robots.txt)
  # Without "only:", all files in priv/static/ are served — robots.txt just works.

Dynamic robots.txt via controller

For environment-based rules (e.g., block all crawlers in staging), serve robots.txt from a controller instead of a static file. Remove the static file first to avoid conflicts.

# lib/my_app_web/controllers/robots_controller.ex
defmodule MyAppWeb.RobotsController do
  use MyAppWeb, :controller

  def index(conn, _params) do
    rules =
      if Application.get_env(:my_app, :env) == :prod do
        """
        User-agent: GPTBot
        Disallow: /

        User-agent: ChatGPT-User
        Disallow: /

        User-agent: ClaudeBot
        Disallow: /

        User-agent: Google-Extended
        Disallow: /

        User-agent: Bytespider
        Disallow: /

        User-agent: CCBot
        Disallow: /

        User-agent: PerplexityBot
        Disallow: /

        User-agent: *
        Allow: /
        """
      else
        # Block everything in staging/dev
        """
        User-agent: *
        Disallow: /
        """
      end

    conn
    |> put_resp_content_type("text/plain")
    |> send_resp(200, rules)
  end
end

# In router.ex:
# get "/robots.txt", RobotsController, :index

2. Hard blocking with a custom Plug

A Plug is a module with two functions: init/1 (compile-time config) and call/2 (request-time logic). Insert it in endpoint.ex after Plug.Static so robots.txt is still served, but all other requests from AI bots get a 403.

lib/my_app_web/plugs/block_ai_bots.ex

defmodule MyAppWeb.Plugs.BlockAiBots do
  @moduledoc "Blocks known AI training crawlers with a 403 response."

  import Plug.Conn

  @behaviour Plug

  # Compiled once at compile time — not per-request
  @blocked_ua_pattern ~r/GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-Web|anthropic-ai|Google-Extended|Bytespider|CCBot|PerplexityBot|Applebot-Extended|DuckAssistBot|cohere-ai|Meta-ExternalAgent|Diffbot|YouBot|Amazonbot/i

  @impl true
  def init(opts), do: opts

  @impl true
  def call(conn, _opts) do
    ua =
      conn
      |> get_req_header("user-agent")
      |> List.first("")

    if Regex.match?(@blocked_ua_pattern, ua) do
      conn
      |> send_resp(403, "Forbidden")
      |> halt()
    else
      conn
    end
  end
end

endpoint.ex — insert after Plug.Static

# lib/my_app_web/endpoint.ex
# ...existing plugs...

plug Plug.Static,
  at: "/",
  from: :my_app,
  gzip: false,
  only: ~w(assets fonts images favicon.ico robots.txt)

# Bot blocking — AFTER Plug.Static so robots.txt is still served
plug MyAppWeb.Plugs.BlockAiBots

# ...rest of endpoint (Plug.RequestId, Plug.Telemetry, etc.)...
plug MyAppWeb.Router

Alternative: block in router.ex pipeline

If you only want to block bots from browser-facing routes (not API endpoints), add the plug to a router pipeline instead of the endpoint.

# lib/my_app_web/router.ex
pipeline :browser do
  plug :accepts, ["html"]
  plug :fetch_session
  plug :fetch_live_flash
  plug :put_root_layout, html: {MyAppWeb.Layouts, :root}
  plug :protect_from_forgery
  plug :put_secure_browser_headers

  # Block AI bots — only for browser routes
  plug MyAppWeb.Plugs.BlockAiBots
end

# API routes are not affected:
pipeline :api do
  plug :accepts, ["json"]
end

3. noai meta tag in HEEx layouts

Phoenix server-renders every page on the initial request — including LiveView pages. The noai meta tag in your root layout is included in every HTML response that reaches the browser, so all crawlers see it.

root.html.heex — site-wide noai (Phoenix 1.7+)

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <meta name="csrf-token" content={get_csrf_token()} />

    <%!-- Opt out of AI training — all pages --%>
    <meta
      name="robots"
      content={assigns[:robots_meta] || "noai, noimageai"}
    />

    <.live_title><%= assigns[:page_title] || "My App" %></.live_title>
    <link phx-track-static rel="stylesheet" href={~p"/assets/app.css"} />
    <script defer phx-track-static src={~p"/assets/app.js"}></script>
  </head>
  <body>
    <%= @inner_content %>
  </body>
</html>

Per-page override via assigns

# In a controller action — override the default noai for a specific page:
def public_post(conn, %{"slug" => slug}) do
  post = Blog.get_post!(slug)

  conn
  |> assign(:robots_meta, "index, follow")  # Allow AI for this page
  |> assign(:page_title, post.title)
  |> render(:show, post: post)
end

# In a LiveView mount — same approach:
def mount(_params, _session, socket) do
  {:ok, assign(socket, robots_meta: "noai, noimageai", page_title: "Dashboard")}
end

4. X-Robots-Tag response header

Add a plug to your :browser pipeline that sets the header on every response. This works even when bots ignore the meta tag.

lib/my_app_web/plugs/x_robots_tag.ex

defmodule MyAppWeb.Plugs.XRobotsTag do
  @moduledoc "Sets X-Robots-Tag header to opt out of AI training."

  import Plug.Conn

  @behaviour Plug

  @impl true
  def init(opts), do: opts

  @impl true
  def call(conn, _opts) do
    put_resp_header(conn, "x-robots-tag", "noai, noimageai")
  end
end

# In router.ex:
pipeline :browser do
  plug :accepts, ["html"]
  plug :fetch_session
  plug :fetch_live_flash
  plug :put_root_layout, html: {MyAppWeb.Layouts, :root}
  plug :protect_from_forgery
  plug :put_secure_browser_headers
  plug MyAppWeb.Plugs.XRobotsTag        # ← Add this
  plug MyAppWeb.Plugs.BlockAiBots       # ← And this
end

The X-Robots-Tag header is set before the bot-blocking plug runs. If the bot is blocked, the 403 response does not include the header (which is fine — a 403 already prevents content access).

5. Complete endpoint + router setup

Combining all layers: Plug.Static serves robots.txt first, the bot-blocking plug rejects known crawlers, the router pipeline adds X-Robots-Tag, and the layout includes the noai meta tag.

endpoint.ex

defmodule MyAppWeb.Endpoint do
  use Phoenix.Endpoint, otp_app: :my_app

  @session_options [
    store: :cookie,
    key: "_my_app_key",
    signing_salt: "your_salt",
    same_site: "Lax"
  ]

  socket "/live", Phoenix.LiveView.Socket,
    websocket: [connect_info: [session: @session_options]]

  # 1. Static files — serves robots.txt before anything else
  plug Plug.Static,
    at: "/",
    from: :my_app,
    gzip: false,
    only: ~w(assets fonts images favicon.ico robots.txt)

  # 2. Bot blocking — after static files so robots.txt is accessible
  plug MyAppWeb.Plugs.BlockAiBots

  plug Plug.RequestId
  plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]

  plug Plug.Parsers,
    parsers: [:urlencoded, :multipart, :json],
    pass: ["*/*"],
    json_decoder: Phoenix.json_library()

  plug Plug.MethodOverride
  plug Plug.Head
  plug Plug.Session, @session_options

  plug MyAppWeb.Router
end

router.ex

defmodule MyAppWeb.Router do
  use MyAppWeb, :router

  pipeline :browser do
    plug :accepts, ["html"]
    plug :fetch_session
    plug :fetch_live_flash
    plug :put_root_layout, html: {MyAppWeb.Layouts, :root}
    plug :protect_from_forgery
    plug :put_secure_browser_headers
    plug MyAppWeb.Plugs.XRobotsTag
  end

  pipeline :api do
    plug :accepts, ["json"]
  end

  scope "/", MyAppWeb do
    pipe_through :browser

    get "/", PageController, :home
    live "/dashboard", DashboardLive
  end

  scope "/api", MyAppWeb do
    pipe_through :api
    # API routes — no bot blocking, no X-Robots-Tag
  end
end

6. nginx — block at the proxy layer

For Phoenix deployed behind nginx (common with Elixir releases and Docker), block AI bots at the proxy before they reach the BEAM VM. This is the most efficient approach — blocked requests never consume Erlang process resources.

# /etc/nginx/conf.d/phoenix-app.conf

map $http_user_agent $blocked_bot {
    default                 0;
    "~*GPTBot"              1;
    "~*ChatGPT-User"        1;
    "~*OAI-SearchBot"       1;
    "~*ClaudeBot"           1;
    "~*anthropic-ai"        1;
    "~*Google-Extended"     1;
    "~*Bytespider"          1;
    "~*CCBot"               1;
    "~*PerplexityBot"       1;
    "~*Applebot-Extended"   1;
}

upstream phoenix {
    server 127.0.0.1:4000;
}

server {
    listen 80;
    server_name example.com;

    # Always serve robots.txt
    location = /robots.txt {
        proxy_pass http://phoenix;
    }

    location / {
        if ($blocked_bot) {
            return 403;
        }
        proxy_pass http://phoenix;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # WebSocket support for LiveView
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
    }
}

7. Deployment comparison

8. Docker with Elixir releases

Phoenix uses mix release to produce a self-contained Erlang release. The priv/static/ directory (including robots.txt) is bundled into the release automatically.

# Dockerfile — multi-stage Phoenix release
FROM elixir:1.16-alpine AS build

RUN apk add --no-cache build-base git

WORKDIR /app

ENV MIX_ENV=prod

COPY mix.exs mix.lock ./
RUN mix deps.get --only prod
RUN mix deps.compile

COPY config config
COPY lib lib
COPY priv priv
COPY assets assets

RUN mix assets.deploy
RUN mix compile
RUN mix release

# ── Runtime ──────────────────────────────────────────────────────────────

FROM alpine:3.19 AS runtime

RUN apk add --no-cache libstdc++ openssl ncurses-libs

WORKDIR /app

COPY --from=build /app/_build/prod/rel/my_app ./

ENV PHX_HOST=example.com
ENV PORT=4000

EXPOSE 4000

CMD ["bin/my_app", "start"]

priv/static/robots.txt is included in the release via mix assets.deploy. Verify with: ls _build/prod/rel/my_app/lib/my_app-0.1.0/priv/static/robots.txt

FAQ