# # robots.txt for jewishpartisans.org # # Default rules apply to all crawlers, including search engines, # AI search/retrieval crawlers, and ADL's Ora crawler. # AI TRAINING crawlers are blocked below. # User-agent: * Crawl-delay: 10 Content-signal: search=yes,ai-input=yes,ai-train=no # CSS, JS, Images Allow: /misc/*.css$ Allow: /misc/*.css? Allow: /misc/*.js$ Allow: /misc/*.js? Allow: /misc/*.gif Allow: /misc/*.jpg Allow: /misc/*.jpeg Allow: /misc/*.png Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png # Directories Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /profiles/ Disallow: /scripts/ Disallow: /themes/ Disallow: /dm/ # Files Disallow: /CHANGELOG.txt Disallow: /cron.php Disallow: /update.php Disallow: /xmlrpc.php # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /user/logout/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/reply/ Disallow: /?q=filter/tips/ Disallow: /?q=node/add/ Disallow: /?q=search/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ Disallow: /?q=user/logout/ # ========================================================= # DISALLOWED - Data harvesters & aggressive scrapers # ========================================================= User-agent: Zoominfobot User-agent: CCBot User-agent: Bytespider User-agent: Amazonbot User-agent: cohere-ai Disallow: / # ========================================================= # DISALLOWED - AI training / extended AI-use crawlers # ========================================================= User-agent: GPTBot User-agent: ClaudeBot User-agent: anthropic-ai User-agent: Google-Extended User-agent: Google-CloudVertexBot User-agent: Applebot-Extended User-agent: Meta-ExternalAgent User-agent: FacebookBot User-agent: DeepseekBot User-agent: xAI-Bot User-agent: GoogleOther User-agent: PanguBot User-agent: TikTokSpider User-agent: cohere-training-data-crawler User-agent: YandexAdditional User-agent: YandexAdditionalBot Disallow: / # ========================================================= # ALLOWED (via User-agent: * above, no separate groups needed) # Search engines: Googlebot, Bingbot, Applebot, Bravebot, Meta-WebIndexer # AI search/retrieval: OAI-SearchBot, ChatGPT-User, ChatGPT Agent, # Claude-SearchBot, Claude-User, Claude-Code, PerplexityBot, # Perplexity-User, Google-Agent, GoogleAgent-URLContext, # Gemini-Deep-Research, NotebookLM, Meta-ExternalFetcher, # DuckAssistBot, MistralAI-User, AzureAI-SearchBot, Amzn-SearchBot # IMPORTANT: adding a named group for any bot drops the default # Disallows for that bot. Copy them in if you do. # ========================================================= Sitemap: https://www.jewishpartisans.org/sitemap.xml