# =============================================================================
# robots.txt — Sophic Labs  (https://sophiclabs.eu)
# Last updated: 2026-10-19
#
# INDEXING POLICY
#   Public pages open for crawling, indexing, and user-facing discovery:
#
#     /              Company homepage
#     /privacy       Privacy Policy
#     /terms         Terms and Conditions
#     /security      Security and Data Handling
#
#   All other routes (API, admin) are off-limits.
#
# DUAL-DOMAIN SETUP
#   sophiclabs.eu   — EU / EEA / UK (this file)
#   sophiclabs.net  — global / rest of world  (served from /robots.txt)
#   Each edition requests a self-referencing canonical. Reciprocal hreflang
#   in HTML and /sitemap.xml describes the intended regional audience.
#   Search engines still decide canonicalisation and indexing.
#
# COPYRIGHT & AI-TRAINING OPT-OUT
#   All content on this site is © Sophic Labs B.V., 2026. All rights reserved.
#
#   PERMITTED:
#     - Crawl, index, cache and store the public pages above for search,
#       retrieval and citation, including AI search and answer engines,
#       provided the original URL / attribution is shown and linked.
#     - Brief quotations and summaries with attribution and a link.
#
#   PROHIBITED (without prior written consent):
#     - Use any content — in whole or in part — to train, fine-tune, or
#       evaluate ML/AI models, or to build evaluation / benchmark or other
#       bulk datasets, commercial or otherwise.
#     - Reproduce, republish, or redistribute content beyond brief quotation.
#     - Mirror or create derivative works from any content.
#
#   Text-and-data-mining rights are reserved (DSM Directive Art. 4(3)):
#   /.well-known/tdmrep.json and the TDM-Reservation response header.
#
#   These prohibitions are enforceable under copyright law, the EU Database
#   Directive (96/9/EC), and Directive (EU) 2013/40.
#
#   Terms and Conditions:  https://sophiclabs.eu/terms
#   Privacy Policy:        https://sophiclabs.eu/privacy
#   Licensing enquiries:   https://sophiclabs.eu/#contact
#
# CHANGES 2026-10-19
#   - Search and AI retrieval/user-fetch agents are allowed (MistralAI-User,
#     Amzn-SearchBot, Amzn-User, Meta-ExternalFetcher, facebookexternalhit);
#     GoogleOther*, AI2Bot*, Amazonbot are blocked as training/R&D.
#   - Allow groups are generated from deploy/bot-policy/allowed-bots.json and
#     the blocked group from deploy/bot-policy/training-bots.json.
#   - Retrieval, caching and citation are permitted; training stays prohibited.
#
# CHANGES 2026-08-11
#   - Removed Section 2 (SEO analysis tools: AhrefsBot, SemrushBot, MJ12bot,
#     DotBot, BLEXBot, Rogerbot, etc.).  These are now hard-blocked at nginx
#     (conf.d/25-bad-bots-slw.conf, TCP 444) — listing them here is redundant.
#   - Added 2026-07-22 ai.robots.txt sync: Claude-Code, Cursor,
#     Google-Gemini-CLI, opencode, Trae, GoogleAgent-URLContext,
#     amazon-QBusiness, Kimi-User, TongyiBot, YiyanBot, and corpus harvesters.
#   - Fixed: file previously contained duplicated/concatenated content.
#   - Removed geo-redirect from nginx; both domains now independently reachable.
#
# REFERENCES
#   https://www.robotstxt.org/
#   https://developers.google.com/search/docs/crawling-indexing/robots/intro
#   https://www.rfc-editor.org/rfc/rfc9309
#   https://darkvisitors.com/agents
#   https://github.com/ai-robots-txt/ai.robots.txt
# =============================================================================


# =============================================================================
# SECTION 1 — Major search-engine crawlers (index, follow)
# =============================================================================
# BEGIN GENERATED search by deploy/bot-policy/generate.mjs from allowed-bots.json. Do not edit.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
User-agent: Google-InspectionTool
User-agent: Google-Site-Verification
User-agent: FeedFetcher-Google
User-agent: AdsBot-Google
User-agent: Bingbot
User-agent: BingPreview
User-agent: adidxbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Ecosia
User-agent: BraveBot
User-agent: Applebot
User-agent: SeznamBot
User-agent: MojeekBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Yeti
# END GENERATED search

Allow: /$
Allow: /privacy
Allow: /terms
Allow: /security
Allow: /assets/
Disallow: /api/
Disallow: /admin/

Crawl-delay: 10


# =============================================================================
# SECTION 2 — AI search, discovery, and retrieval assistants
#
# These agents power user-facing AI search features, answer engines, and
# retrieval-augmented interfaces.  Public marketing pages are open for
# indexing, retrieval, and surfacing in AI-generated answers.
# Attribution to the original URL is required.
# =============================================================================

# BEGIN GENERATED ai by deploy/bot-policy/generate.mjs from allowed-bots.json. Do not edit.
# -- OpenAI search & user-facing agents ---------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Operator

# -- Anthropic search & retrieval ---------------------------------------------
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web

# -- Google AI discovery & deep research --------------------------------------
User-agent: Gemini-Deep-Research
User-agent: Google-NotebookLM
User-agent: NotebookLM
User-agent: GoogleAgent-Mariner
User-agent: Google-Agent
User-agent: Google-CloudVertexBot
User-agent: CloudVertexBot
User-agent: Google-Firebase

# -- AI-powered search engines ------------------------------------------------
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: YouBot
User-agent: ExaBot
User-agent: TavilyBot
User-agent: kagi-fetcher
User-agent: DuckAssistBot
User-agent: PhindBot
User-agent: iAskBot
User-agent: iaskspider
User-agent: iaskspider/2.0
User-agent: Andibot
User-agent: LinkupBot
User-agent: LinerBot

# -- Amazon and Mistral search / user-fetch agents ----------------------------
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: MistralAI-User

# -- Link-preview and user-fetch agents (not training) ------------------------
User-agent: Meta-ExternalFetcher
User-agent: facebookexternalhit

# -- Enterprise AI search infrastructure --------------------------------------
User-agent: AzureAI-SearchBot
User-agent: Cloudflare-AutoRAG
# END GENERATED ai

Allow: /$
Allow: /privacy
Allow: /terms
Allow: /security
Allow: /assets/
Disallow: /api/
Disallow: /admin/

Crawl-delay: 15


# =============================================================================
# SECTION 3 — Blocked crawlers
#
# AI model training crawlers, bulk data scrapers, agentic automation frameworks,
# and AI coding agents that fetch arbitrary URLs.  The list is generated from
# deploy/bot-policy/training-bots.json, which also drives nginx and the API.
# Disallow: / applies to the ENTIRE site — no exceptions.
#
# NOTE: SEO analysis tools (AhrefsBot, SemrushBot, MJ12bot, DotBot, BLEXBot,
# Rogerbot, ScreamingFrog, etc.) are hard-blocked at nginx (TCP 444) and not
# listed here — they will never reach robots.txt parsing.
# =============================================================================

# BEGIN GENERATED training by deploy/bot-policy/generate.mjs from training-bots.json. Do not edit.
User-agent: GPTBot
User-agent: OpenAI
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: FacebookBot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: CCBot
User-agent: archive.org_bot
User-agent: ia_archiver
User-agent: cohere-training-data-crawler
User-agent: MistralAI
User-agent: xAIBot
User-agent: DeepSeekBot
User-agent: Kimi-User
User-agent: TongyiBot
User-agent: YiyanBot
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: AmazonBuyForMe
User-agent: bedrockbot
User-agent: NovaAct
User-agent: amazon-QBusiness
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: AI2Bot-DeepResearchEval
User-agent: img2dataset
User-agent: LAIONDownloader
User-agent: laion-huggingface-processor
User-agent: Claude-Code
User-agent: Code
User-agent: Cursor
User-agent: Google-Gemini-CLI
User-agent: opencode
User-agent: Trae
User-agent: GoogleAgent-URLContext
User-agent: ApifyBot
User-agent: ApifyWebsiteContentCrawler
User-agent: Crawl4AI
User-agent: Devin
User-agent: FirecrawlAgent
User-agent: Manus-User
User-agent: TwinAgent
User-agent: Diffbot
User-agent: magpie-crawler
User-agent: Omgili
User-agent: Omgilibot
User-agent: ImagesiftBot
User-agent: DataForSeoBot
User-agent: Brightbot
User-agent: webz.io
User-agent: Webzio-Extended
User-agent: TurnitinBot
User-agent: Awario
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Meltwater
User-agent: Kangaroo Bot
User-agent: Barkrowler
User-agent: QuillBot
User-agent: quillbot.com
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
User-agent: Scrapy
User-agent: atlassian-bot
User-agent: AddSearchBot
User-agent: aiHitBot
User-agent: Anomura
User-agent: Aranet-SearchBot
User-agent: bigsur.ai
User-agent: BuddyBot
User-agent: Channel3Bot
User-agent: Cotoyogi
User-agent: Crawlspace
User-agent: FriendlyCrawler
User-agent: IbouBot
User-agent: imageSpider
User-agent: KlaviyoAIBot
User-agent: MyCentralAIScraperBot
User-agent: newsai
User-agent: Panscient
User-agent: panscient.com
User-agent: Poggio-Citations
User-agent: QualifiedBot
User-agent: ShapBot
User-agent: Thinkbot
User-agent: VelenPublicWebCrawler
User-agent: WRTNBot
User-agent: AgentTimes
User-agent: AIWebIndex
User-agent: CragCrawler
User-agent: GeistHaus-PageFetcher
User-agent: HenkBot
User-agent: Mozilla-Tabstack
User-agent: Querit-SearchBot
User-agent: QueritBot
User-agent: Shap-User
User-agent: Terra Cotta
User-agent: UseAI
Disallow: /
# END GENERATED training


# =============================================================================
# SECTION 4 — All other crawlers (catch-all default)
# =============================================================================
User-agent: *

Allow: /$
Allow: /privacy
Allow: /terms
Allow: /security
Allow: /assets/
Disallow: /api/
Disallow: /admin/

Crawl-delay: 10


# =============================================================================
# Sitemap
# =============================================================================
Sitemap: https://sophiclabs.eu/sitemap.xml