HOW TO DETECT AI BOTS ON YOUR WEBSITE

# /etc/nginx/conf.d/ai-bot-detection.conf
map $http_user_agent $is_ai_bot {
    default         0;
    ~*GPTBot        1;
    ~*ChatGPT-User  1;
    ~*OAI-SearchBot 1;
    ~*ClaudeBot     1;
    ~*claude-web    1;
    ~*PerplexityBot 1;
    ~*Bytespider    1;
    ~*Google-Extended 1;
    ~*CCBot         1;
    ~*meta-externalagent 1;
    ~*cohere-ai     1;
    ~*anthropic-ai  1;
}

server {
    # Log AI bots separately
    access_log /var/log/nginx/ai-bots.log combined if=$is_ai_bot;
    
    # Optional: rate limit AI bots
    limit_req_zone $is_ai_bot zone=ai_bots:10m rate=10r/s;
}

// middleware.ts
import { NextRequest, NextResponse } from 'next/server'

const AI_BOT_PATTERNS = [
  /GPTBot/i, /ChatGPT-User/i, /OAI-SearchBot/i,
  /ClaudeBot/i, /claude-web/i, /anthropic-ai/i,
  /PerplexityBot/i, /Perplexity-User/i,
  /Google-Extended/i, /Bytespider/i,
  /CCBot/i, /meta-externalagent/i, /cohere-ai/i,
]

export function middleware(request: NextRequest) {
  const ua = request.headers.get('user-agent') || ''
  const isAIBot = AI_BOT_PATTERNS.some(p => p.test(ua))
  
  if (isAIBot) {
    // Log, track, or report to NORAD
    console.log(`AI Bot detected: ${ua.slice(0, 100)}`)
    
    // Optional: report to NORAD
    fetch('https://api.clawbotden.com/api/v1/public/norad/ingest', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        bot_type: ua.match(AI_BOT_PATTERNS.find(p => p.test(ua))!)?.[0] || 'Unknown',
        user_agent: ua.slice(0, 500),
        decision: 'allow',
        page_url: request.nextUrl.href,
      }),
    }).catch(() => {})
  }
  
  return NextResponse.next()
}

#!/bin/bash
# Verify if an IP belongs to a known AI crawler
IP="66.249.66.1"

# Reverse DNS lookup
HOST=$(dig -x $IP +short)
echo "Reverse DNS: $HOST"

# Forward DNS verification
if [[ "$HOST" == *"googlebot.com." ]] || [[ "$HOST" == *"google.com." ]]; then
    FORWARD=$(dig $HOST +short)
    if [[ "$FORWARD" == "$IP" ]]; then
        echo "✅ Verified Googlebot"
    else
        echo "❌ Spoofed Googlebot"
    fi
elif [[ "$HOST" == *"openai.com." ]]; then
    echo "✅ Verified OpenAI crawler"
elif [[ "$HOST" == *"search.msn.com." ]]; then
    echo "✅ Verified Bingbot"
else
    echo "⚠️ Unknown origin: $HOST"
fi