{
 "title": "Russian websites and AI crawlers: robots.txt, llms.txt and Organization markup (September 2026)",
 "measured": "2026-09-25",
 "publisher": "SKY INCOM (https://www.skyname.net)",
 "source_page": "https://www.skyname.net/blog/ai-crawlers-ru-2026/",
 "license": "CC BY 4.0",
 "sample": "Top 2,000 .ru and .xn--p1ai domains in the Tranco top-1M list (list ID K9PXW); 1292 of them served an HTML homepage (HTTP 200) and form the sample. CDN, API and technical domains without a website are excluded.",
 "method": [
  "GET https:// (fallback http://) homepage; JSON-LD presence and an Organization-type @type (Organization, Corporation, LocalBusiness, NewsMediaOrganization, OnlineStore, OnlineBusiness) detected in the first 400 KB.",
  "GET /robots.txt on the final homepage origin; a bot counts as blocked when the group that applies to it (its own user-agent group, otherwise *) contains \"Disallow: /\" and contains no Allow rule at all. Partial access (Disallow: / plus Allow for some paths) counts as not blocked.",
  "Bot shares are computed over 1258 sites that do not block Googlebot or YandexBot, so that sites closed to all crawlers do not inflate the AI figures.",
  "GET /llms.txt; counted when HTTP 200, non-HTML, non-binary text of 30+ characters. llms_txt_markdown = file starts with \"#\" (the llms.txt format).",
  "Single measurement from one location with a disclosed research user-agent; sites that refuse unknown user-agents are excluded as having no website. Blocking at firewall or CDN level is not visible."
 ],
 "stats": {
  "domains_checked": 2000,
  "sites": 1292,
  "robots_txt": 1074,
  "open_to_search": 1258,
  "closed_all": 34,
  "blocked_share_among_open": {
   "GPTBot": [
    48,
    3.8
   ],
   "OAI-SearchBot": [
    8,
    0.6
   ],
   "ChatGPT-User": [
    29,
    2.3
   ],
   "ClaudeBot": [
    22,
    1.7
   ],
   "PerplexityBot": [
    9,
    0.7
   ],
   "Google-Extended": [
    11,
    0.9
   ],
   "CCBot": [
    36,
    2.9
   ],
   "Bytespider": [
    36,
    2.9
   ]
  },
  "any_ai_blocked": [
   70,
   5.6
  ],
  "all_ai_blocked": [
   6,
   0.5
  ],
  "llms_text": [
   106,
   8.2
  ],
  "llms_md": [
   99,
   7.7
  ],
  "jsonld": [
   342,
   26.5
  ],
  "org_schema": [
   237,
   18.3
  ]
 }
}