How to Block AI Bots in Node.js Restify

1. Bot detection

Pure JavaScript, no dependencies. Array.prototype.some() short-circuits on first match. String.prototype.includes() for literal substring matching.

// bot-utils.js — AI bot detection, no external dependencies
'use strict';

const AI_BOT_PATTERNS = [
  'gptbot',
  'chatgpt-user',
  'claudebot',
  'anthropic-ai',
  'ccbot',
  'google-extended',
  'cohere-ai',
  'meta-externalagent',
  'bytespider',
  'omgili',
  'diffbot',
  'imagesiftbot',
  'magpie-crawler',
  'amazonbot',
  'dataprovider',
  'netcraft',
];

/**
 * Returns true if ua matches a known AI crawler pattern.
 * String.prototype.includes() — literal substring match, no regex.
 * toLowerCase() normalises before comparison.
 * @param {string} ua
 * @returns {boolean}
 */
function isAiBot(ua) {
  if (!ua) return false;
  const lower = ua.toLowerCase();
  return AI_BOT_PATTERNS.some(pattern => lower.includes(pattern));
}

module.exports = { isAiBot };

2. Middleware — function(req, res, next)

Restify middleware is identical in signature to Express. All Express middleware is compatible. req.headers['user-agent'] returns undefined when absent — use || ''. Call res.send(403, 'Forbidden') to block; never call next() after it.

// middleware/bot-blocker.js — Restify global middleware
'use strict';

const { isAiBot } = require('../bot-utils');

/**
 * Restify middleware signature: function(req, res, next)
 * Identical to Express — all Express middleware is compatible with Restify.
 *
 * req.headers keys are ALWAYS lowercase in Node.js (IncomingMessage normalises them).
 * req.headers['user-agent'] returns undefined when absent — use || '' for safety.
 *
 * res.send(statusCode, body) — Restify enhanced send:
 *   - Sets status code
 *   - Serialises body (strings → text/plain, objects → JSON)
 *   - Ends the response — do NOT call next() after res.send()
 *
 * next() — continues to the next middleware or route handler.
 * next(false) — stops the chain without sending a response (not needed here).
 */
function botBlockerMiddleware(req, res, next) {
  // Path guard: robots.txt must be reachable so bots can read Disallow rules.
  if (req.path() === '/robots.txt') {
    return next();
  }

  // req.headers['user-agent'] — lowercase key, returns undefined when absent.
  const ua = req.headers['user-agent'] || '';

  if (isAiBot(ua)) {
    // Block: set headers then send — res.send() ends the response.
    // Do NOT call next() after res.send() — response is already sent.
    res.header('X-Robots-Tag', 'noai, noimageai');
    res.header('Content-Type', 'text/plain');
    return res.send(403, 'Forbidden');
  }

  // Pass: inject X-Robots-Tag on the way through, then continue.
  res.header('X-Robots-Tag', 'noai, noimageai');
  return next();
}

module.exports = { botBlockerMiddleware };

3. Server setup — server.use()

Register the bot blocker as the first server.use() call — it runs before built-in plugins and route handlers. Note that server.use() only fires for matched routes; see server.pre() below for full coverage.

// server.js — Restify server with global bot-blocking middleware
'use strict';

const restify = require('restify');
const { botBlockerMiddleware } = require('./middleware/bot-blocker');

const server = restify.createServer({ name: 'my-api' });

// server.use() — runs AFTER routing (only for matched routes).
// Register the bot blocker before any other use() middleware.
server.use(botBlockerMiddleware);

// Built-in Restify plugins — body parser, query parser, etc.
server.use(restify.plugins.bodyParser());
server.use(restify.plugins.queryParser());

// robots.txt — accessible to bots (middleware passes it through).
server.get('/robots.txt', (req, res) => {
  res.header('Content-Type', 'text/plain');
  res.send(200, `User-agent: *
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /
`);
});

server.get('/', (req, res) => {
  res.send({ message: 'Hello' });  // objects are auto-serialised to JSON
});

server.get('/api/data', (req, res) => {
  res.send({ data: 'value' });
});

server.listen(8080, () => {
  console.log(`${server.name} listening on ${server.url}`);
});

4. server.pre() vs server.use()

server.pre() fires before route matching — it runs for every request including paths that would 404. This is the more thorough approach for bot blocking: a bot probing unknown URLs (common crawler behaviour) bypasses server.use() but not server.pre().

// server.pre() — pre-routing middleware.
// Fires BEFORE Restify matches the request to a route.
// Runs for ALL requests including paths that would return 404.
// Use server.pre() for more thorough bot blocking.

server.pre(function(req, res, next) {
  // This runs even for unknown paths — bots hitting arbitrary URLs are blocked.
  if (req.path() === '/robots.txt') {
    return next();
  }

  const ua = req.headers['user-agent'] || '';
  if (isAiBot(ua)) {
    res.header('X-Robots-Tag', 'noai, noimageai');
    return res.send(403, 'Forbidden');
  }

  res.header('X-Robots-Tag', 'noai, noimageai');
  return next();
});

// vs server.use() — only runs for routes that Restify matched.
// If a bot requests /unknown-path, server.use() middleware is SKIPPED
// (Restify returns 404 before middleware runs for unmatched routes).
// server.pre() does NOT have this limitation.

5. Route-level inline middleware

Pass middleware functions before the route handler to scope protection to specific routes. Useful when most routes are public but sensitive API endpoints need bot blocking.

// Route-level inline middleware — protect specific routes only.
// Pass middleware function(s) before the route handler.

const { botBlockerMiddleware } = require('./middleware/bot-blocker');

// Single route — only /api/data is bot-blocked.
server.get('/api/data', botBlockerMiddleware, (req, res) => {
  res.send({ data: 'value' });
});

// Multiple inline middleware — chain runs left to right.
server.get('/api/premium', botBlockerMiddleware, authMiddleware, (req, res) => {
  res.send({ premium: true });
});

// Public route — no bot blocker.
server.get('/', (req, res) => {
  res.send({ message: 'Hello' });
});

Key points

• Header names are always lowercase in Node.js: req.headers is Node.js's IncomingMessage.headers — all keys are normalised to lowercase. req.headers['user-agent'] works; req.headers['User-Agent'] returns undefined. This is true in Express, Fastify, Hapi, and any Node.js http server.
• server.pre() is more thorough than server.use() for bot blocking: server.use() only runs for requests that match a registered route. Bots often probe arbitrary paths — those would return 404 and bypass server.use() middleware. server.pre() runs before routing and catches everything.
• Do not call next() after res.send(): res.send() in Restify ends the response. Calling next() afterwards causes a "headers already sent" error and attempts to continue the middleware chain with an already-completed response.
• Express middleware is directly compatible: Restify implements the same (req, res, next) signature as Express. Any Express middleware — rate limiters, auth helpers, loggers — works with server.use() in Restify without modification.
• res.send() auto-serialises objects to JSON: Passing an object to res.send() serialises it to JSON and sets Content-Type: application/json. Passing a string sets text/plain. For the 403 body, use a plain string — res.send(403, 'Forbidden').
• req.path() vs req.url: Restify's req.path() returns just the pathname without query string (e.g. /robots.txt). req.url includes the query string (e.g. /robots.txt?foo=bar). Use req.path() for path guards.

Framework comparison — Node.js REST frameworks

Restify's server.pre() is its most distinctive feature for bot blocking — it provides pre-routing middleware that Express lacks natively (Express achieves the same by registering app.use() before route registration). Fastify's onRequest hook and Hapi's request lifecycle both run before routing, making them equivalent to server.pre(). Restify's middleware signature is Express-compatible — the botBlockerMiddleware function above works in Express without modification.