Monitor AI Bot Traffic
Know What's Crawling You

# OpenAI (3 bots, 3 purposes)
GPTBot          # Training data for GPT models
ChatGPT-User    # Real-time browsing when users ask ChatGPT to read a URL
OAI-SearchBot   # Indexing for ChatGPT Search results

# Anthropic (2 tokens)
ClaudeBot       # Training data for Claude models
anthropic-ai    # Alternative Anthropic identifier

# Google (3 AI-specific bots)
Google-Extended         # Gemini/Bard AI training
Gemini-Deep-Research    # Gemini Advanced deep research feature
Google-NotebookLM       # NotebookLM URL ingestion

# Meta
meta-externalagent  # Llama training (NOT the link preview bot)

# Microsoft
bingbot         # Powers both Bing Search AND Copilot answers

# ByteDance
Bytespider      # TikTok / Doubao AI training

# Amazon (3 bots)
Amazonbot           # AI training for Alexa/Nova
Amzn-SearchBot      # Rufus AI shopping assistant
Amzn-User           # Live query browsing

# AI Search Engines
PerplexityBot       # Perplexity AI search indexing
YouBot              # You.com AI assistant
DuckAssistBot       # DuckDuckGo AI answers

# European AI
MistralBot      # Mistral AI (Le Chat, Mixtral)
DeepSeekBot     # DeepSeek (V3, R1)

# xAI
xAI-Bot         # Grok training (Elon Musk's xAI)

# Data Brokers (feed multiple AI companies)
CCBot           # Common Crawl → GPT, Gemini, Llama, Mistral, etc.
Diffbot         # Commercial data broker → Llama, Mistral, DiffbotLLM
omgili          # Webz.io data broker
omgilibot       # Webz.io (alternative token)
webzio-extended # Webz.io AI training crawler

# Apple
Applebot-Extended   # Apple Intelligence training

# Academic
AI2Bot          # Allen Institute (Semantic Scholar)
Ai2Bot-Dolma    # Allen Institute (OLMo training dataset)

# Enterprise
cohere-ai       # Cohere (Command R, Embed)

#!/bin/bash
# ai-bot-report.sh — Scan nginx access logs for AI bot traffic

LOG="/var/log/nginx/access.log"

# All known AI bot tokens (case-insensitive grep)
AI_BOTS="GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|Google-NotebookLM|Bytespider|PerplexityBot|meta-externalagent|Amazonbot|Amzn-SearchBot|CCBot|Diffbot|cohere-ai|DeepSeekBot|xAI-Bot|MistralBot|YouBot|DuckAssistBot|AI2Bot|Ai2Bot-Dolma|Applebot-Extended|omgili|omgilibot|webzio-extended"

echo "=== AI Bot Traffic Report ==="
echo "Log: $LOG"
echo "Date: $(date)"
echo ""

# Total AI bot requests
TOTAL=$(grep -ciE "$AI_BOTS" "$LOG")
ALL=$(wc -l < "$LOG")
echo "AI bot requests: $TOTAL / $ALL total ($(echo "scale=1; $TOTAL * 100 / $ALL" | bc)%)"
echo ""

# Breakdown by bot
echo "=== Requests per bot ==="
for bot in GPTBot ChatGPT-User OAI-SearchBot ClaudeBot Bytespider PerplexityBot Google-Extended meta-externalagent Amazonbot CCBot Diffbot DeepSeekBot xAI-Bot MistralBot cohere-ai; do
  COUNT=$(grep -ci "$bot" "$LOG")
  if [ "$COUNT" -gt 0 ]; then
    echo "  $bot: $COUNT"
  fi
done
echo ""

# Top 20 pages hit by AI bots
echo "=== Top 20 pages targeted by AI bots ==="
grep -iE "$AI_BOTS" "$LOG" \
  | awk '{print $7}' \
  | sort | uniq -c | sort -rn \
  | head -20
echo ""

# Hourly distribution (when do they crawl?)
echo "=== Hourly distribution ==="
grep -iE "$AI_BOTS" "$LOG" \
  | awk '{print substr($4,14,2)":00"}' \
  | sort | uniq -c | sort -rn \
  | head -10

# Add to crontab (crontab -e):
# Run daily at midnight, email results
0 0 * * * /opt/scripts/ai-bot-report.sh > /tmp/ai-bot-daily.txt 2>&1

# Or append to a rolling log:
0 0 * * * /opt/scripts/ai-bot-report.sh >> /var/log/ai-bot-reports.log 2>&1

# For Apache logs, change the LOG path:
# LOG="/var/log/apache2/access.log"
# Format is slightly different but grep works the same

http {
  # Map AI bot user agents to a flag
  map $http_user_agent $is_ai_bot {
    default 0;
    "~*GPTBot"              1;
    "~*ChatGPT-User"        1;
    "~*OAI-SearchBot"       1;
    "~*ClaudeBot"           1;
    "~*anthropic-ai"        1;
    "~*Google-Extended"      1;
    "~*Bytespider"          1;
    "~*PerplexityBot"       1;
    "~*meta-externalagent"  1;
    "~*Amazonbot"           1;
    "~*CCBot"               1;
    "~*Diffbot"             1;
    "~*DeepSeekBot"         1;
    "~*xAI-Bot"             1;
    "~*MistralBot"          1;
    "~*cohere-ai"           1;
    "~*YouBot"              1;
    "~*DuckAssistBot"       1;
    "~*Applebot-Extended"   1;
    "~*AI2Bot"              1;
    "~*Ai2Bot-Dolma"        1;
  }

  # JSON log format for easy parsing
  log_format ai_bot_json escape=json
    '{'
      '"time":"$time_iso8601",'
      '"ip":"$remote_addr",'
      '"method":"$request_method",'
      '"path":"$uri",'
      '"status":$status,'
      '"ua":"$http_user_agent",'
      '"bytes":$body_bytes_sent,'
      '"referer":"$http_referer"'
    '}';

  server {
    # Write AI bot requests to a separate file
    access_log /var/log/nginx/ai-bots.log ai_bot_json if=$is_ai_bot;

    # Normal access log (all traffic)
    access_log /var/log/nginx/access.log;
  }
}

import { NextRequest, NextResponse } from 'next/server';

const AI_BOT_PATTERNS = [
  /GPTBot/i, /ChatGPT-User/i, /OAI-SearchBot/i,
  /ClaudeBot/i, /anthropic-ai/i,
  /Google-Extended/i, /Gemini-Deep-Research/i,
  /Bytespider/i, /PerplexityBot/i,
  /meta-externalagent/i, /Amazonbot/i,
  /CCBot/i, /Diffbot/i, /cohere-ai/i,
  /DeepSeekBot/i, /xAI-Bot/i, /MistralBot/i,
  /YouBot/i, /DuckAssistBot/i,
  /Applebot-Extended/i, /AI2Bot/i,
];

function detectAIBot(ua: string): string | null {
  for (const pattern of AI_BOT_PATTERNS) {
    const match = ua.match(pattern);
    if (match) return match[0];
  }
  return null;
}

export function middleware(req: NextRequest) {
  const ua = req.headers.get('user-agent') || '';
  const bot = detectAIBot(ua);

  if (bot) {
    // Log to your preferred backend
    // Option 1: Console log (appears in Vercel Function Logs)
    console.log(JSON.stringify({
      type: 'ai_bot',
      bot,
      path: req.nextUrl.pathname,
      ip: req.headers.get('x-forwarded-for'),
      timestamp: new Date().toISOString(),
    }));

    // Option 2: Send to analytics endpoint (non-blocking)
    // fetch('https://your-analytics.com/api/bot-hit', {
    //   method: 'POST',
    //   body: JSON.stringify({ bot, path: req.nextUrl.pathname }),
    // }).catch(() => {}); // Fire and forget

    // Option 3: Add custom header for downstream processing
    const res = NextResponse.next();
    res.headers.set('x-ai-bot', bot);
    return res;
  }

  return NextResponse.next();
}

export const config = {
  matcher: ['/((?!_next|favicon.ico|robots.txt|sitemap.xml).*)'],
};

#!/bin/bash
# ai-bot-monitor.sh — Watch AI bot log in real-time
# Usage: ./ai-bot-monitor.sh

LOG="/var/log/nginx/ai-bots.log"
WEBHOOK="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

echo "🔍 Monitoring AI bot traffic..."
echo "   Log: $LOG"
echo "   Press Ctrl+C to stop"
echo ""

tail -f "$LOG" | while read -r line; do
  BOT=$(echo "$line" | jq -r '.ua' | grep -oiE 'GPTBot|ClaudeBot|Bytespider|PerplexityBot|Google-Extended|DeepSeekBot|xAI-Bot|meta-externalagent|Amazonbot|CCBot|Diffbot')
  PATH_HIT=$(echo "$line" | jq -r '.path')
  IP=$(echo "$line" | jq -r '.ip')
  TIME=$(echo "$line" | jq -r '.time')

  # Print to terminal
  echo "[$TIME] $BOT → $PATH_HIT (from $IP)"

  # Alert on new/unusual bots (optional)
  # Uncomment to send Slack alerts:
  # curl -s -X POST "$WEBHOOK" \
  #   -H 'Content-type: application/json' \
  #   -d "{\"text\":\"🤖 AI Bot: $BOT hit $PATH_HIT\"}" \
  #   > /dev/null
done