{
  "title": "AI Crawler Statistics 2026: Traffic, Training, Referrals and Blocking",
  "url": "https://statisticsglobal.org/ai-crawler-statistics/",
  "updated": "2026-11-12",
  "licence": "https://statisticsglobal.org/data-licence/",
  "attribution": "Source: [publisher] via StatisticsGlobal (https://statisticsglobal.org/ai-crawler-statistics/)",
  "statistics": [
    {
      "id": "aic-cf-training-2026",
      "sentence": "52% of crawler requests on Cloudflare's network were for AI training in June 2026, up from 22% in spring 2025, according to Cloudflare.",
      "value": 52,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "Content Independence Day, one year on: building the business model for the agentic Internet",
      "source_url": "https://blog.cloudflare.com/agentic-internet-bot-report/",
      "source_published": "1 July 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/cloudflare/agentic-internet-bot-report-2026-11-12.csv",
      "sample": "Cloudflare Radar and Cloudflare Investor Day 2026 data. Crawler purpose is classified by Cloudflare. Cloudflare says more than 20% of the web sits behind its network. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-training-2026"
    },
    {
      "id": "aic-cf-ratio-anthropic-2025",
      "sentence": "Anthropic crawled about 38,066 pages for every visitor it referred in July 2025, down from 286,930 in January 2025, according to Cloudflare.",
      "value": 38065.7,
      "unit": "crawls per referral",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ratio-anthropic-2025"
    },
    {
      "id": "aic-tb-eu-ratio-2026",
      "sentence": "European publishers received one human referral for every 179 AI bot scrapes in H1 2026, over 3 times worse than North American sites, according to TollBit.",
      "value": 179,
      "unit": "scrapes per referral",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-eu-ratio-2026"
    },
    {
      "id": "aic-cf-ai-html-2025",
      "sentence": "AI bots other than Googlebot made an average of 4.2% of HTML requests on Cloudflare's network in 2025, according to Cloudflare.",
      "value": 4.2,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ai-html-2025"
    },
    {
      "id": "aic-ri-openai-2023",
      "sentence": "48% of top news websites in 10 countries blocked OpenAI's crawlers by the end of 2023, according to the Reuters Institute.",
      "value": 48,
      "unit": "%",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-openai-2023"
    },
    {
      "id": "aic-cf-googlebot-html-2025",
      "sentence": "Googlebot alone made 4.5% of HTML requests on Cloudflare's network in 2025, according to Cloudflare.",
      "value": 4.5,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-googlebot-html-2025"
    },
    {
      "id": "aic-calc-ai-plus-googlebot-html",
      "sentence": "AI bots and Googlebot together made 8.7% of HTML requests on Cloudflare's network in 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 8.7,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "4.2% (AI bots excluding Googlebot) + 4.5% (Googlebot) = 8.7%. Cloudflare reports the two shares separately because Googlebot crawls for both search and AI.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ai-plus-googlebot-html"
    },
    {
      "id": "aic-cf-human-html-2025",
      "sentence": "Humans generated 47% of HTML requests on Cloudflare's network on 2 December 2025, and non-AI bots 44%, according to Cloudflare.",
      "value": 47,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-human-html-2025"
    },
    {
      "id": "aic-cf-crawl-growth-2025",
      "sentence": "AI and search crawler traffic grew 18% between May 2024 and May 2025 on a fixed set of Cloudflare customers, according to Cloudflare.",
      "value": 18,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-crawl-growth-2025"
    },
    {
      "id": "aic-cf-googlebot-growth-2025",
      "sentence": "Googlebot crawl requests grew 96% between May 2024 and May 2025, according to Cloudflare.",
      "value": 96,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-googlebot-growth-2025"
    },
    {
      "id": "aic-cf-ai-verified-2025",
      "sentence": "AI crawlers generated 20% of verified bot traffic on Cloudflare's network in 2025, half the 40% from search engine crawlers, according to Cloudflare.",
      "value": 20,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ai-verified-2025"
    },
    {
      "id": "aic-fy-bots-2026",
      "sentence": "Bots made 49% of requests to sites on Fastly's network in January 2026, and humans 51%, according to Fastly.",
      "value": 49,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-bots-2026"
    },
    {
      "id": "aic-fy-origin-bots-2026",
      "sentence": "Bots made 60% of requests that reached origin servers on Fastly's network in January 2026, according to Fastly.",
      "value": 60,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-origin-bots-2026"
    },
    {
      "id": "aic-fy-ai-wanted-2026",
      "sentence": "AI bots made about 8% of wanted bot traffic on Fastly's network in January 2026, according to Fastly.",
      "value": 8,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-ai-wanted-2026"
    },
    {
      "id": "aic-calc-fy-ai-all",
      "sentence": "AI bots classed as wanted made roughly 0.08% of all requests on Fastly's network in January 2026, calculated by StatisticsGlobal from Fastly data.",
      "value": 0.08,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Our analysis",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "Wanted (verified) bots 1% of all requests x AI share of wanted bot traffic 8% = 0.08%. Both inputs are rounded, so treat as an order of magnitude. Unverified AI scrapers fall in Fastly's unwanted bot category and are not included.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-fy-ai-all"
    },
    {
      "id": "aic-fy-publishing-ai-2026",
      "sentence": "AI bots made 4% of wanted bot traffic to publishing sites on Fastly's network in January 2026, according to Fastly.",
      "value": 4,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-publishing-ai-2026"
    },
    {
      "id": "aic-cf-gptbot-ai-share-2025",
      "sentence": "GPTBot had a 30% share of AI-only crawler traffic on Cloudflare's network in May 2025, up from 5% in May 2024, according to Cloudflare.",
      "value": 30,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-gptbot-ai-share-2025"
    },
    {
      "id": "aic-calc-gptbot-share-pp",
      "sentence": "GPTBot's share of AI-only crawler traffic rose 25 percentage points, a sixfold increase, between May 2024 and May 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 25,
      "unit": "percentage points",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "30% (May 2025) - 5% (May 2024) = 25 points; 30 / 5 = 6x. Cloudflare AI-only crawler ranking table.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-gptbot-share-pp"
    },
    {
      "id": "aic-cf-meta-ai-share-2025",
      "sentence": "Meta-ExternalAgent took a 19% share of AI-only crawler traffic in May 2025, according to Cloudflare.",
      "value": 19,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-meta-ai-share-2025"
    },
    {
      "id": "aic-cf-bytespider-ai-share-2025",
      "sentence": "ByteDance's Bytespider had a 7.2% share of AI-only crawler traffic in May 2025, down from 42% in May 2024, according to Cloudflare.",
      "value": 7.2,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-bytespider-ai-share-2025"
    },
    {
      "id": "aic-calc-bytespider-pp",
      "sentence": "Bytespider's share of AI-only crawler traffic fell 34.8 percentage points between May 2024 and May 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 34.8,
      "unit": "percentage points",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "42% (May 2024) - 7.2% (May 2025) = 34.8 points. Cloudflare AI-only crawler ranking table.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-bytespider-pp"
    },
    {
      "id": "aic-cf-gptbot-ai-jul-2025",
      "sentence": "GPTBot had a 28.1% share of AI-only bot traffic in July 2025, up from 11.9% in July 2024, according to Cloudflare.",
      "value": 28.1,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-gptbot-ai-jul-2025"
    },
    {
      "id": "aic-cf-claudebot-ai-jul-2025",
      "sentence": "ClaudeBot had a 23.3% share of AI-only bot traffic in July 2025, up from 15% in July 2024, according to Cloudflare.",
      "value": 23.3,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-claudebot-ai-jul-2025"
    },
    {
      "id": "aic-cf-googlebot-share-2025",
      "sentence": "Googlebot made 50% of all AI and search crawler requests in May 2025, up from 30% in May 2024, according to Cloudflare.",
      "value": 50,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-googlebot-share-2025"
    },
    {
      "id": "aic-cf-googlebot-jul-2025",
      "sentence": "Googlebot made 39% of all AI and search crawler traffic in July 2025, according to Cloudflare.",
      "value": 39,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-googlebot-jul-2025"
    },
    {
      "id": "aic-cf-googlebot-verified-2025",
      "sentence": "Googlebot generated over 28% of verified bot traffic on Cloudflare's network in 2025, according to Cloudflare.",
      "value": 28,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-googlebot-verified-2025"
    },
    {
      "id": "aic-cf-gptbot-verified-2025",
      "sentence": "GPTBot generated about 7.5% of verified bot traffic on Cloudflare's network in 2025, the second highest of any bot, according to Cloudflare.",
      "value": 7.5,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-gptbot-verified-2025"
    },
    {
      "id": "aic-cf-perplexity-growth-2025",
      "sentence": "PerplexityBot crawl requests rose 157,490% between May 2024 and May 2025, from a very low base, according to Cloudflare.",
      "value": 157490,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-perplexity-growth-2025"
    },
    {
      "id": "aic-cf-chatgptuser-growth-2025",
      "sentence": "ChatGPT-User requests rose 2,825% between May 2024 and May 2025, reaching 1.3% of AI and search crawling, according to Cloudflare.",
      "value": 2825,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-chatgptuser-growth-2025"
    },
    {
      "id": "aic-fy-googleother-2026",
      "sentence": "GoogleOther made 36% of AI crawler traffic on Fastly's network in January 2026, according to Fastly.",
      "value": 36,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-googleother-2026"
    },
    {
      "id": "aic-fy-meta-2026",
      "sentence": "Meta's AI crawler made 28% of AI crawler traffic on Fastly's network in January 2026, according to Fastly.",
      "value": 28,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-meta-2026"
    },
    {
      "id": "aic-fy-petalbot-2026",
      "sentence": "Huawei's PetalBot made 18% of AI crawler traffic on Fastly's network in January 2026, according to Fastly.",
      "value": 18,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-petalbot-2026"
    },
    {
      "id": "aic-calc-fy-top3",
      "sentence": "GoogleOther, Meta's AI crawler and PetalBot together made 82% of AI crawler traffic on Fastly's network in January 2026, calculated by StatisticsGlobal from Fastly data.",
      "value": 82,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Our analysis",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "36% + 28% + 18% = 82%. Fastly Threat Insights Report 26Q1, AI crawler traffic.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-fy-top3"
    },
    {
      "id": "aic-cf-europe-gptbot-2025",
      "sentence": "GPTBot made 10.8% of crawler traffic to Europe-based Cloudflare customers in October 2025, behind Googlebot and Bingbot, according to Cloudflare.",
      "value": 10.8,
      "unit": "%",
      "place": "Europe",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-europe-gptbot-2025"
    },
    {
      "id": "aic-cf-asia-gptbot-2025",
      "sentence": "GPTBot made 14.0% of crawler traffic to Asia-based Cloudflare customers in October 2025, second only to Googlebot, according to Cloudflare.",
      "value": 14,
      "unit": "%",
      "place": "Asia",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-asia-gptbot-2025"
    },
    {
      "id": "aic-cf-na-googlebot-2025",
      "sentence": "Googlebot made 45.5% of crawler traffic to North America-based Cloudflare customers in October 2025, according to Cloudflare.",
      "value": 45.5,
      "unit": "%",
      "place": "North America",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-na-googlebot-2025"
    },
    {
      "id": "aic-cf-industry-2025",
      "sentence": "Retail and computer software sites drew just over 40% of AI crawler traffic on Cloudflare's network in October 2025, according to Cloudflare.",
      "value": 40,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-industry-2025"
    },
    {
      "id": "aic-cf-ratio-openai-2025",
      "sentence": "OpenAI crawled about 1,091 pages for every visitor it referred in July 2025, according to Cloudflare.",
      "value": 1091.4,
      "unit": "crawls per referral",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ratio-openai-2025"
    },
    {
      "id": "aic-cf-ratio-perplexity-2025",
      "sentence": "Perplexity crawled about 195 pages for every visitor it referred in July 2025, up from 54.6 in January 2025, according to Cloudflare.",
      "value": 194.8,
      "unit": "crawls per referral",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ratio-perplexity-2025"
    },
    {
      "id": "aic-cf-ratio-microsoft-2025",
      "sentence": "Microsoft crawled 40.7 pages for every visitor it referred in July 2025, according to Cloudflare.",
      "value": 40.7,
      "unit": "crawls per referral",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ratio-microsoft-2025"
    },
    {
      "id": "aic-cf-ratio-google-2025",
      "sentence": "Google crawled 5.4 pages for every visitor it referred in July 2025, according to Cloudflare.",
      "value": 5.4,
      "unit": "crawls per referral",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-ratio-google-2025"
    },
    {
      "id": "aic-calc-ratio-anthropic-openai",
      "sentence": "Anthropic's crawl-to-referral ratio was 34.9 times OpenAI's in July 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 34.9,
      "unit": "x",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "38,065.7 / 1,091.4 = 34.88, rounded to 34.9. Cloudflare Radar crawl-to-refer ratios for July 2025.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ratio-anthropic-openai"
    },
    {
      "id": "aic-calc-ratio-openai-perplexity",
      "sentence": "OpenAI's crawl-to-referral ratio was 5.6 times Perplexity's in July 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 5.6,
      "unit": "x",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "1,091.4 / 194.8 = 5.60. Cloudflare Radar crawl-to-refer ratios for July 2025.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ratio-openai-perplexity"
    },
    {
      "id": "aic-calc-ratio-ai-vs-google",
      "sentence": "OpenAI crawled 202 times as many pages per referral as Google in July 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 202,
      "unit": "x",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "1,091.4 / 5.4 = 202.1, rounded to 202. Cloudflare Radar crawl-to-refer ratios for July 2025.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ratio-ai-vs-google"
    },
    {
      "id": "aic-tb-ratio-q1-2026",
      "sentence": "The AI scrape-to-referral ratio on European publisher sites was 150 to 1 in Q1 2026, according to TollBit.",
      "value": 150,
      "unit": "scrapes per referral",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-ratio-q1-2026"
    },
    {
      "id": "aic-tb-ratio-q2-2026",
      "sentence": "The AI scrape-to-referral ratio on European publisher sites was 227 to 1 in Q2 2026, according to TollBit.",
      "value": 227,
      "unit": "scrapes per referral",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-ratio-q2-2026"
    },
    {
      "id": "aic-calc-tb-ratio-change",
      "sentence": "The AI scrape-to-referral ratio on European publisher sites worsened 51% from Q1 to Q2 2026, calculated by StatisticsGlobal from TollBit data.",
      "value": 51,
      "unit": "%",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Our analysis",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "(227 / 150) - 1 = 0.513, rounded to 51%. TollBit Q1 2026 and Q2 2026 ratios.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-tb-ratio-change"
    },
    {
      "id": "aic-tb-eu-human-2026",
      "sentence": "European publisher sites saw about 1 AI bot scrape for every 33 human visits in H1 2026, according to TollBit.",
      "value": 33,
      "unit": "human visits per AI scrape",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-eu-human-2026"
    },
    {
      "id": "aic-cf-google-referrals-2026",
      "sentence": "Google accounted for about 88% of referral traffic to websites in 2026, according to Cloudflare.",
      "value": 88,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "Content Independence Day, one year on: building the business model for the agentic Internet",
      "source_url": "https://blog.cloudflare.com/agentic-internet-bot-report/",
      "source_published": "1 July 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/cloudflare/agentic-internet-bot-report-2026-11-12.csv",
      "sample": "Cloudflare Radar and Cloudflare Investor Day 2026 data. Crawler purpose is classified by Cloudflare. Cloudflare says more than 20% of the web sits behind its network. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-google-referrals-2026"
    },
    {
      "id": "aic-calc-training-2026-pp",
      "sentence": "The AI training share of crawler requests rose 30 percentage points between spring 2025 and June 2026, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 30,
      "unit": "percentage points",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "Content Independence Day, one year on: building the business model for the agentic Internet",
      "source_url": "https://blog.cloudflare.com/agentic-internet-bot-report/",
      "source_published": "1 July 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/cloudflare/agentic-internet-bot-report-2026-11-12.csv",
      "sample": "Cloudflare Radar and Cloudflare Investor Day 2026 data. Crawler purpose is classified by Cloudflare. Cloudflare says more than 20% of the web sits behind its network. Cloudflare sells AI crawler controls.",
      "method": "52% (June 2026) - 22% (spring 2025) = 30 points. Share of all crawler requests, a wider base than the AI-only crawling shares in the August 2025 post.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-training-2026-pp"
    },
    {
      "id": "aic-cf-mixed-2026",
      "sentence": "Mixed-use crawlers that blend search, agent use and training made over 36% of crawler activity in 2026, according to Cloudflare.",
      "value": 36,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "Content Independence Day, one year on: building the business model for the agentic Internet",
      "source_url": "https://blog.cloudflare.com/agentic-internet-bot-report/",
      "source_published": "1 July 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/cloudflare/agentic-internet-bot-report-2026-11-12.csv",
      "sample": "Cloudflare Radar and Cloudflare Investor Day 2026 data. Crawler purpose is classified by Cloudflare. Cloudflare says more than 20% of the web sits behind its network. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-mixed-2026"
    },
    {
      "id": "aic-cf-training-share-2025",
      "sentence": "Training made up 79% of AI crawling in July 2025, up from 72% in July 2024, according to Cloudflare.",
      "value": 79,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-training-share-2025"
    },
    {
      "id": "aic-calc-training-pp",
      "sentence": "Training's share of AI crawling rose 7 percentage points between July 2024 and July 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 7,
      "unit": "percentage points",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "79% (July 2025) - 72% (July 2024) = 7 points. Cloudflare crawl purpose data.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-training-pp"
    },
    {
      "id": "aic-cf-search-share-2025",
      "sentence": "Search made up 17% of AI crawling in July 2025, down from 26% in July 2024, according to Cloudflare.",
      "value": 17,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-search-share-2025"
    },
    {
      "id": "aic-cf-useraction-share-2025",
      "sentence": "User actions made up 3.2% of AI crawling in July 2025, up from 2% in July 2024, according to Cloudflare.",
      "value": 3.2,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-useraction-share-2025"
    },
    {
      "id": "aic-calc-useraction-growth",
      "sentence": "The user action share of AI crawling grew 60% between July 2024 and July 2025, from 2% to 3.2%, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 60,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "The crawl-to-click gap: Cloudflare data on AI bots, training, and referrals",
      "source_url": "https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "source_published": "29 August 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261104023915/https://blog.cloudflare.com/crawlers-click-ai-bots-training/",
      "sample": "Cloudflare Radar data on AI and search crawler requests and referrals across the Cloudflare network, July 2024 to July 2025. Crawl purpose is classified from operator disclosures. Cloudflare sells AI crawler controls.",
      "method": "(3.2 / 2) - 1 = 0.60, or 60%. Shares of AI crawling, Cloudflare crawl purpose data.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-useraction-growth"
    },
    {
      "id": "aic-cf-useraction-growth-2025",
      "sentence": "AI user action crawling grew more than 15 times over during 2025, according to Cloudflare.",
      "value": 15,
      "unit": "x",
      "place": "Worldwide",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "The 2025 Cloudflare Radar Year in Review",
      "source_url": "https://blog.cloudflare.com/radar-2025-year-in-review/",
      "source_published": "15 December 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261112045117/https://blog.cloudflare.com/radar-2025-year-in-review/",
      "sample": "Request traffic across the Cloudflare network in 2025, data to early December 2025. HTML request shares classify traffic as human, AI bot or non-AI bot. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-useraction-growth-2025"
    },
    {
      "id": "aic-fy-fetch-openai-2026",
      "sentence": "OpenAI's ChatGPT-User and SearchBot made 97% of AI fetcher traffic on Fastly's network in January 2026, according to Fastly.",
      "value": 97,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-fetch-openai-2026"
    },
    {
      "id": "aic-fy-fetch-origin-2026",
      "sentence": "57% of AI fetcher requests on Fastly's network went for content that was not cached in January 2026, according to Fastly.",
      "value": 57,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Fastly",
      "source_title": "Fastly Threat Insights Report 26Q1: AI, Bots, and the Agentic Future of the Web",
      "source_url": "https://learn.fastly.com/rs/025-XKO-469/images/Fastly-Threat-Insights-Report-26Q1.pdf",
      "source_published": "April 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/fastly/threat-insights-26q1-2026-11-12.csv",
      "sample": "Traffic seen by Fastly Next-Gen WAF and Bot Management, 1 to 31 January 2026. Fastly says these products protect over 130,000 applications and APIs. Search engine crawlers are not counted as AI bots. Fastly sells bot management.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-fy-fetch-origin-2026"
    },
    {
      "id": "aic-cf-robots-ai-rules-2025",
      "sentence": "About 14% of top 10,000 domains with a robots.txt file had rules targeting AI bots in June 2025, 546 of 3,816 domains, according to Cloudflare.",
      "value": 14,
      "unit": "%",
      "place": "Top 10,000 domains",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-robots-ai-rules-2025"
    },
    {
      "id": "aic-cf-gptbot-disallow-2025",
      "sentence": "GPTBot was disallowed by 312 top 10,000 domains in June 2025, 250 fully and 62 partially, the most of any bot, according to Cloudflare.",
      "value": 312,
      "unit": "domains",
      "place": "Top 10,000 domains",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-gptbot-disallow-2025"
    },
    {
      "id": "aic-calc-gptbot-disallow-share",
      "sentence": "8.2% of top 10,000 domains with a robots.txt file disallowed GPTBot in June 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 8.2,
      "unit": "%",
      "place": "Top 10,000 domains",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "312 domains disallowing GPTBot / 3,816 domains with a robots.txt file = 8.18%, rounded to 8.2%. Full disallows only: 250 / 3,816 = 6.6%.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-gptbot-disallow-share"
    },
    {
      "id": "aic-cf-gptbot-allow-2025",
      "sentence": "GPTBot was explicitly allowed by 61 top 10,000 domains in June 2025, 18 fully and 43 partially, according to Cloudflare.",
      "value": 61,
      "unit": "domains",
      "place": "Top 10,000 domains",
      "data_year": "2025",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-gptbot-allow-2025"
    },
    {
      "id": "aic-calc-gptbot-block-allow",
      "sentence": "Top 10,000 domains disallowed GPTBot 5.1 times as often as they explicitly allowed it in June 2025, calculated by StatisticsGlobal from Cloudflare data.",
      "value": 5.1,
      "unit": "to 1",
      "place": "Top 10,000 domains",
      "data_year": "2025",
      "source_type": "Our analysis",
      "publisher": "Cloudflare",
      "source_title": "From Googlebot to GPTBot: who's crawling your site in 2025",
      "source_url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "source_published": "1 July 2025, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261027180241/https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025/",
      "sample": "Cloudflare network traffic from AI and search crawlers, matched by user agent, May 2024 vs May 2025, using a fixed customer cohort. robots.txt data from the Cloudflare Radar top 10,000 domains, 6 June 2025. Cloudflare sells AI crawler controls.",
      "method": "312 domains with a GPTBot disallow / 61 domains with a GPTBot allow = 5.11, rounded to 5.1.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-gptbot-block-allow"
    },
    {
      "id": "aic-dpi-c4-tokens-2024",
      "sentence": "Over 5% of all tokens in the C4 AI training dataset were fully restricted by robots.txt by 2024, according to the Data Provenance Initiative.",
      "value": 5,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2024",
      "source_type": "Research",
      "publisher": "Data Provenance Initiative",
      "source_title": "Consent in Crisis: The Rapid Decline of the AI Data Commons (arXiv 2407.14933)",
      "source_url": "https://arxiv.org/abs/2407.14933",
      "source_published": "July 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261103075400/https://arxiv.org/abs/2407.14933",
      "sample": "Longitudinal audit of robots.txt and terms of service for about 14,000 web domains behind the C4, RefinedWeb and Dolma AI training corpora, 2023 to 2024.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-dpi-c4-tokens-2024"
    },
    {
      "id": "aic-dpi-c4-critical-2024",
      "sentence": "Over 28% of the most actively maintained, critical sources in C4 were fully restricted by robots.txt by 2024, according to the Data Provenance Initiative.",
      "value": 28,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2024",
      "source_type": "Research",
      "publisher": "Data Provenance Initiative",
      "source_title": "Consent in Crisis: The Rapid Decline of the AI Data Commons (arXiv 2407.14933)",
      "source_url": "https://arxiv.org/abs/2407.14933",
      "source_published": "July 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261103075400/https://arxiv.org/abs/2407.14933",
      "sample": "Longitudinal audit of robots.txt and terms of service for about 14,000 web domains behind the C4, RefinedWeb and Dolma AI training corpora, 2023 to 2024.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-dpi-c4-critical-2024"
    },
    {
      "id": "aic-dpi-c4-tos-2024",
      "sentence": "45% of the C4 AI training dataset was restricted by terms of service crawling rules by 2024, according to the Data Provenance Initiative.",
      "value": 45,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2024",
      "source_type": "Research",
      "publisher": "Data Provenance Initiative",
      "source_title": "Consent in Crisis: The Rapid Decline of the AI Data Commons (arXiv 2407.14933)",
      "source_url": "https://arxiv.org/abs/2407.14933",
      "source_published": "July 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261103075400/https://arxiv.org/abs/2407.14933",
      "sample": "Longitudinal audit of robots.txt and terms of service for about 14,000 web domains behind the C4, RefinedWeb and Dolma AI training corpora, 2023 to 2024.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-dpi-c4-tos-2024"
    },
    {
      "id": "aic-tb-robots-2026",
      "sentence": "The median European site's robots.txt rules were ignored 2.8 times more often than North American sites' in H1 2026, according to TollBit.",
      "value": 2.8,
      "unit": "x",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-robots-2026"
    },
    {
      "id": "aic-tb-commoncrawl-2026",
      "sentence": "Common Crawl scraped about 74% of sites on the TollBit network in H1 2026, roughly double its reach from Q1 to Q2, according to TollBit.",
      "value": 74,
      "unit": "%",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-commoncrawl-2026"
    },
    {
      "id": "aic-tb-uk-scraped-2026",
      "sentence": "19 of the top 20 UK sites ranked by Press Gazette could be scraped by at least one of 14 tested scraping tools in 2026, according to TollBit.",
      "value": 19,
      "unit": "of 20 sites",
      "place": "United Kingdom",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-uk-scraped-2026"
    },
    {
      "id": "aic-tb-eu-scrapes-2026",
      "sentence": "Median AI scrapes per site were 4 times higher on European sites than North American sites in H1 2026, according to TollBit.",
      "value": 4,
      "unit": "x",
      "place": "Europe",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "TollBit",
      "source_title": "State of the Bots 2026 Q1 & Q2: The Bad Bots",
      "source_url": "https://tollbit.com/bots/",
      "source_published": "2026, covers January to June 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/tollbit/state-of-the-bots-2026-h1-2026-11-12.csv",
      "sample": "Server log data from publisher sites on the TollBit network, H1 2026. Counts only bots that identify themselves by user agent. UK scraper test covered the top 20 UK sites and 14 scraping tools. TollBit sells AI content licensing and bot paywalls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-tb-eu-scrapes-2026"
    },
    {
      "id": "aic-cf-deals-2026",
      "sentence": "More than 50 publisher-AI content agreements had been signed between 2023 and July 2026, according to Cloudflare.",
      "value": 50,
      "unit": "agreements",
      "place": "Worldwide",
      "data_year": "2026",
      "source_type": "Vendor report",
      "publisher": "Cloudflare",
      "source_title": "Content Independence Day, one year on: building the business model for the agentic Internet",
      "source_url": "https://blog.cloudflare.com/agentic-internet-bot-report/",
      "source_published": "1 July 2026, accessed 12 November 2026",
      "archive_url": "https://statisticsglobal.org/data/snapshots/cloudflare/agentic-internet-bot-report-2026-11-12.csv",
      "sample": "Cloudflare Radar and Cloudflare Investor Day 2026 data. Crawler purpose is classified by Cloudflare. Cloudflare says more than 20% of the web sits behind its network. Cloudflare sells AI crawler controls.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-cf-deals-2026"
    },
    {
      "id": "aic-ri-google-2023",
      "sentence": "24% of top news websites in 10 countries blocked Google's AI crawler by the end of 2023, according to the Reuters Institute.",
      "value": 24,
      "unit": "%",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-google-2023"
    },
    {
      "id": "aic-calc-ri-openai-google",
      "sentence": "Top news websites were twice as likely to block OpenAI's crawlers as Google's AI crawler in 2023, calculated by StatisticsGlobal from Reuters Institute data.",
      "value": 2,
      "unit": "x",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Our analysis",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "48% blocking OpenAI / 24% blocking Google AI = 2.0. Reuters Institute end-2023 figures across 10 countries.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ri-openai-google"
    },
    {
      "id": "aic-ri-us-2023",
      "sentence": "79% of top US news websites blocked OpenAI's crawlers by the end of 2023, the highest of 10 countries, according to the Reuters Institute.",
      "value": 79,
      "unit": "%",
      "place": "United States",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-us-2023"
    },
    {
      "id": "aic-ri-mx-pl-2023",
      "sentence": "20% of top news websites in Mexico and in Poland blocked OpenAI's crawlers by the end of 2023, the lowest of 10 countries, according to the Reuters Institute.",
      "value": 20,
      "unit": "%",
      "place": "Mexico and Poland",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-mx-pl-2023"
    },
    {
      "id": "aic-calc-ri-country-gap",
      "sentence": "The share of top news sites blocking OpenAI was 59 percentage points higher in the US than in Mexico or Poland in 2023, calculated by StatisticsGlobal from Reuters Institute data.",
      "value": 59,
      "unit": "percentage points",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Our analysis",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "79% (US) - 20% (Mexico, Poland) = 59 points. Reuters Institute end-2023 figures.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ri-country-gap"
    },
    {
      "id": "aic-ri-de-google-2023",
      "sentence": "60% of top German news websites blocked Google's AI crawler by the end of 2023, the highest of 10 countries, according to the Reuters Institute.",
      "value": 60,
      "unit": "%",
      "place": "Germany",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-de-google-2023"
    },
    {
      "id": "aic-ri-print-2023",
      "sentence": "57% of legacy print news websites blocked OpenAI's crawlers by the end of 2023, against 31% of digital-born outlets, according to the Reuters Institute.",
      "value": 57,
      "unit": "%",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-print-2023"
    },
    {
      "id": "aic-calc-ri-print-gap",
      "sentence": "Legacy print news sites were 26 percentage points more likely than digital-born outlets to block OpenAI's crawlers in 2023, calculated by StatisticsGlobal from Reuters Institute data.",
      "value": 26,
      "unit": "percentage points",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Our analysis",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "57% (legacy print) - 31% (digital-born) = 26 points. Reuters Institute end-2023 figures across 10 countries.",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-calc-ri-print-gap"
    },
    {
      "id": "aic-ri-overlap-2023",
      "sentence": "97% of top news websites that blocked Google's AI crawler in 2023 also blocked OpenAI's crawlers, according to the Reuters Institute.",
      "value": 97,
      "unit": "%",
      "place": "10 countries",
      "data_year": "2023",
      "source_type": "Research",
      "publisher": "Reuters Institute",
      "source_title": "How many news websites block AI crawlers? (Reuters Institute for the Study of Journalism, University of Oxford)",
      "source_url": "https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "source_published": "22 February 2024, accessed 12 November 2026",
      "archive_url": "http://web.archive.org/web/20261102141002/https://reutersinstitute.politics.ox.ac.uk/how-many-news-websites-block-ai-crawlers",
      "sample": "robots.txt files of the 15 most widely used online news sources in each of 10 countries (Brazil, Denmark, Germany, India, Mexico, Norway, Poland, Spain, UK, US), read from Wayback Machine archives for every available day of 2023.",
      "method": "",
      "verified_on": "2026-11-12",
      "url": "https://statisticsglobal.org/ai-crawler-statistics/#stat-aic-ri-overlap-2023"
    }
  ]
}