{
 "study": "cdn-blocking-ai-crawlers",
 "what_this_is": "The firm's own record behind the field note: the 2026-08-20 finding on jiashley.com, the Cloudflare zone settings before and after the fix, the audit tool's robots.txt checks across the firm's properties and one prospect, and the live robots.txt as served on the day this file was built.",
 "finding_2026_08_20": {
  "site": "jiashley.com",
  "recorded_in": "phaneia/FINDINGS.md, entry '2026-08-20 \u2014 Cloudflare was silently blocking AI crawlers on jiashley.com'",
  "zone_settings_before": {
   "ai_bots_protection": "block",
   "is_robots_txt_managed": true
  },
  "crawlers_disallowed_by_the_served_robots_txt": [
   "GPTBot",
   "ClaudeBot",
   "Google-Extended",
   "CCBot",
   "Amazonbot",
   "Applebot-Extended"
  ],
  "origin_robots_txt_policy": "AI crawlers welcome",
  "fix": {
   "api": "PUT /zones/{id}/bot_management",
   "body": {
    "ai_bots_protection": "disabled",
    "is_robots_txt_managed": false
   }
  },
  "audit_before_fix": "WARN, 3 AI bots blocked (recorded in the finding; the pre-fix JSON was not kept)",
  "audit_after_fix": "PASS, AI bots blocked: none (jiashley.com_2026-08-20.json)",
  "record_quote": "The zone had `ai_bots_protection = block` plus Cloudflare's managed robots.txt enabled. Net effect: GPTBot, ClaudeBot, Google-Extended, CCBot, Amazonbot, and Applebot-Extended were all disallowed AND blocked at the network edge. The origin robots.txt said \"AI crawlers welcome\"; the edge said the opposite, and the edge wins."
 },
 "audit_robots_checks": [
  {
   "site": "an agency selling AI search optimization (name withheld)",
   "date": "2026-08-21",
   "status": "WARN",
   "detail": "present; AI bots blocked: ['GPTBot', 'ClaudeBot', 'Google-Extended']"
  },
  {
   "site": "anamnesis.smtry.ai",
   "date": "2026-08-20",
   "status": "FAIL",
   "detail": "missing or soft-404 (HTML served)"
  },
  {
   "site": "anamnesis.smtry.ai",
   "date": "2026-08-20",
   "status": "PASS",
   "detail": "present; AI bots blocked: none"
  },
  {
   "site": "jiashley.com",
   "date": "2026-08-20",
   "status": "FAIL",
   "detail": "missing or soft-404 (HTML served)"
  },
  {
   "site": "jiashley.com",
   "date": "2026-08-20",
   "status": "PASS",
   "detail": "present; AI bots blocked: none"
  },
  {
   "site": "jiashley.com",
   "date": "2026-08-21",
   "status": "PASS",
   "detail": "present; AI bots blocked: none"
  },
  {
   "site": "jiashley.com",
   "date": "2026-08-25",
   "status": "PASS",
   "detail": "present; AI bots blocked: none"
  },
  {
   "site": "smtry.ai",
   "date": "2026-08-20",
   "status": "FAIL",
   "detail": "missing or soft-404 (HTML served)"
  },
  {
   "site": "smtry.ai",
   "date": "2026-08-20",
   "status": "PASS",
   "detail": "present; AI bots blocked: none"
  }
 ],
 "live_capture": {
  "site": "jiashley.com",
  "fetched": "2026-09-05",
  "ai_crawlers_disallowed_now": [],
  "robots_txt": "# jiashley.com \u2014 public consulting site; search and AI crawlers welcome.\n# Bulk scrapers that hoard content without attribution are not.\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://jiashley.com/sitemap.xml"
 },
 "method": {
  "tool": "phaneia/siteaudit.py robots.txt check: fetch /robots.txt from outside, flag AI crawlers under Disallow: /",
  "note": "Audit JSON files record the tool's verdict lines, not the served file body; the served body is captured here for the live check only."
 }
}