# Trooth - https://trooth.co
#
# This file is the source of truth for Trooth's crawler policy. /ai.txt and
# /llms.txt restate it in prose and are held to agreeing with it by a build
# gate (scripts/check-crawler-policy.mjs), so if they ever disagree with this
# file, this file is right.
#
# The short version. Training crawlers may read Trooth's own pages and may not
# read the company records under /network/ and /trust/. Search crawlers and
# user-initiated fetchers may read everything public, including the records,
# because that is how a buyer's question gets a truthful answer. There is also
# a public, read-only MCP endpoint that needs no key and is a better way to ask
# than crawling: https://api.trooth.co/public/mcp
#
# Every user-agent named below is spelled as its operator spells it, and the
# page that documents it is listed here:
#
#   GPTBot                 developers.openai.com/api/docs/bots
#   ClaudeBot              privacy.claude.com/en/articles/8896518
#   Google-Extended        developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers
#   Applebot-Extended      support.apple.com/en-us/119829
#   Amazonbot              developer.amazon.com/amazonbot
#   meta-externalagent     developers.facebook.com/docs/sharing/webmasters/web-crawlers/
#   MistralAI-Training     docs.mistral.ai/robots
#   CCBot                  commoncrawl.org/ccbot
#   Bytespider             developers.cloudflare.com/ai-crawl-control/reference/bots/ (third party; ByteDance publishes none)
#   ImagesiftBot           imagesift.com/about (operator page)
#   DiffBot                diffbot.com (operator page)
#   OAI-SearchBot          developers.openai.com/api/docs/bots
#   ChatGPT-User           developers.openai.com/api/docs/bots
#   Claude-SearchBot       privacy.claude.com/en/articles/8896518
#   Claude-User            privacy.claude.com/en/articles/8896518
#   PerplexityBot          docs.perplexity.ai/guides/bots
#   Perplexity-User        docs.perplexity.ai/guides/bots
#   MistralAI-Index        docs.mistral.ai/robots
#   MistralAI-User         docs.mistral.ai/robots
#   DuckAssistBot          duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
#   Amzn-SearchBot         developer.amazon.com/amazonbot
#   Amzn-User              developer.amazon.com/amazonbot
#   meta-webindexer        developers.facebook.com/docs/sharing/webmasters/web-crawlers/
#   meta-externalfetcher   developers.facebook.com/docs/sharing/webmasters/web-crawlers/
#   Applebot               support.apple.com/en-us/119829
#   Googlebot              developers.google.com/search/docs/appearance/ai-features
#   bingbot                bing.com/webmasters
#
# Trooth does not use customer data to train AI models, and does not sell or
# share customer data for others to train on.

# --------------------------------------------------------------------------
# Model-training crawlers. Trooth's own pages, yes. The companies' records, no.
# --------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: MistralAI-Training
User-agent: CCBot
User-agent: Bytespider
User-agent: ImagesiftBot
User-agent: DiffBot
Disallow: /network/
Disallow: /trust/
Disallow: /verify/
Disallow: /api/
Disallow: /dashboard
Disallow: /admin
Disallow: /buyers/dashboard
Disallow: /auditor/
Disallow: /welcome
Disallow: /onboarding
Allow: /api/og/
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# --------------------------------------------------------------------------
# Search indexes and user-initiated fetchers. Everything public, including the
# company records. This is the group that decides whether a Trooth profile is
# in the answer when somebody asks an assistant about a vendor.
# --------------------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: MistralAI-Index
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: meta-webindexer
User-agent: meta-externalfetcher
User-agent: Applebot
User-agent: Googlebot
User-agent: bingbot
Allow: /
Allow: /api/og/
Disallow: /api/
Disallow: /dashboard
Disallow: /admin
Disallow: /buyers/dashboard
Disallow: /auditor/
Disallow: /welcome
Disallow: /onboarding
Content-Signal: search=yes, ai-input=yes, ai-train=no

# --------------------------------------------------------------------------
# Everyone else.
# --------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /api/og/
Disallow: /api/
Disallow: /dashboard
Disallow: /admin
Disallow: /buyers/dashboard
Disallow: /auditor/
Disallow: /welcome
Disallow: /onboarding
Content-Signal: search=yes, ai-input=yes, ai-train=no

Host: https://trooth.co
Sitemap: https://api.trooth.co/sitemap.xml
Sitemap: https://trooth.co/sitemap.xml