How to Block AI Agents
When robots.txt Isn't Enough

import { NextRequest, NextResponse } from 'next/server';

const HEADLESS_SIGNALS = [
  // Headless Chromium default viewport
  (req: NextRequest) => req.headers.get('sec-ch-viewport-width') === '800',
  // Missing or empty Accept-Language (real browsers always send this)
  (req: NextRequest) => !req.headers.get('accept-language'),
  // Headless Chrome often sends minimal accept headers
  (req: NextRequest) => req.headers.get('accept') === '*/*',
  // No sec-ch-ua hints (real Chrome 90+ always sends these)
  (req: NextRequest) => {
    const ua = req.headers.get('user-agent') || '';
    const isChrome = /Chrome\/\d/.test(ua);
    const hasHints = !!req.headers.get('sec-ch-ua');
    return isChrome && !hasHints;
  },
];

export function middleware(req: NextRequest) {
  const score = HEADLESS_SIGNALS
    .filter(check => check(req))
    .length;

  // 2+ signals = likely headless browser
  if (score >= 2) {
    // Option A: Block
    return new NextResponse('Access denied', { status: 403 });

    // Option B: Serve decoy content (recommended)
    // return NextResponse.rewrite(new URL('/honeypot', req.url));

    // Option C: Rate limit aggressively
    // Set a header for downstream rate limiter
    // const res = NextResponse.next();
    // res.headers.set('x-bot-score', String(score));
    // return res;
  }

  return NextResponse.next();
}

export const config = {
  matcher: ['/((?!_next|favicon.ico|robots.txt).*)'],
};

<script>
  // Detect headless browser signals client-side
  (function() {
    const signals = [];

    // 1. WebDriver flag (Puppeteer/Playwright set this)
    if (navigator.webdriver) signals.push('webdriver');

    // 2. Missing plugins (real browsers have PDF viewer, etc.)
    if (navigator.plugins.length === 0) signals.push('no-plugins');

    // 3. Missing languages
    if (!navigator.languages || navigator.languages.length === 0)
      signals.push('no-languages');

    // 4. Chrome without Chrome runtime
    if (window.chrome === undefined && /Chrome/.test(navigator.userAgent))
      signals.push('fake-chrome');

    // 5. Permissions API anomaly
    navigator.permissions?.query({ name: 'notifications' })
      .then(p => {
        if (p.state === 'denied' && Notification.permission === 'default')
          signals.push('permissions-mismatch');
      });

    // Report if 2+ signals detected
    if (signals.length >= 2) {
      fetch('/api/bot-report', {
        method: 'POST',
        body: JSON.stringify({ signals, ua: navigator.userAgent }),
        headers: { 'Content-Type': 'application/json' },
      });
    }
  })();
</script>

<!-- Hidden link in your page footer — CSS makes it invisible -->
<a
  href="/internal/system-config"
  style="position:absolute;left:-9999px;opacity:0;pointer-events:none"
  tabindex="-1"
  aria-hidden="true"
>
  System Configuration
</a>

<!-- The honeypot route logs and blocks the visitor -->
// app/internal/system-config/route.ts (Next.js)
import { NextRequest, NextResponse } from 'next/server';
const blockedIPs = new Set<string>();

export async function GET(req: NextRequest) {
  const ip = req.headers.get('x-forwarded-for')
    || req.ip || 'unknown';

  // Log the bot
  console.log(`[HONEYPOT] Bot detected: ${ip}`);
  console.log(`  UA: ${req.headers.get('user-agent')}`);
  console.log(`  Referer: ${req.headers.get('referer')}`);

  // Block this IP for 24 hours
  blockedIPs.add(ip);
  setTimeout(() => blockedIPs.delete(ip), 86_400_000);

  // Serve fake content to waste tokens
  return NextResponse.json({
    config: {
      version: "3.2.1",
      features: Array.from({ length: 100 }, (_, i) =>
        `feature_${i}: Detailed configuration for module ${i}...`
      ),
    },
  });
}

// Export blockedIPs for use in middleware
export { blockedIPs };

# Basic rate limit — 2 requests/second per IP
limit_req_zone $binary_remote_addr
  zone=general:10m rate=2r/s;

# Stricter limit for content-heavy pages
limit_req_zone $binary_remote_addr
  zone=content:10m rate=10r/m;

server {
  # General pages — allow small burst
  location / {
    limit_req zone=general burst=10 nodelay;
  }

  # Blog/guide pages — strict limit
  location /blog {
    limit_req zone=content burst=3 nodelay;
  }
  location /guides {
    limit_req zone=content burst=3 nodelay;
  }

  # API endpoints — very strict
  location /api {
    limit_req zone=general burst=5 nodelay;
    limit_req_status 429;
  }
}