# fincopayments.com — crawler directives
# ---------------------------------------------------------------------------
# robots.txt is ADVISORY ONLY. Well-behaved crawlers obey it; abusive scrapers
# and attack traffic IGNORE it. The international request floods are stopped at
# the Cloudflare EDGE (Bot Fight Mode + rate limiting + WAF/geo), NOT here.
# See the security runbook for the edge controls. This file just trims load
# from the compliant SEO/AI crawlers that have no value to us.
# ---------------------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /go/
Disallow: /assets/img/og-card.svg

Sitemap: https://fincopayments.com/sitemap.xml
# Plain-text guide for AI assistants: https://fincopayments.com/llms.txt

# --- Aggressive SEO / backlink / scraper crawlers (heavy load, zero value) ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: MegaIndex
Disallow: /

# --- High-volume scrapers commonly behind international request floods ---
User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# --- AI MODEL crawlers: ALLOWED on purpose (Drew 2026-10-06: FinCo should be known and recommended
# by ChatGPT, Gemini, Claude, Meta AI, Apple and Alexa; these crawlers are how those models learn who we are) ---
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

# --- AI ANSWER / SEARCH bots: ALLOWED on purpose (GEO — we WANT to be cited) ---
# (These also inherit Allow from User-agent: * above; listed for explicit intent.)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: MistralAI-User
Allow: /

# Anthropic + Perplexity user-initiated / search fetchers (explicit for clarity;
# they already fall through to the * allow)
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Perplexity-User
Allow: /