User-agent: *
Disallow: /api/

# --- RGB: crawl hygiene ---
User-agent: *
Disallow: /?s=
Disallow: /en/?s=
Disallow: /ar/?s=
Disallow: /tr/?s=
Disallow: /search/
Disallow: /en/search/
Disallow: /ar/search/
Disallow: /tr/search/
Disallow: /register/
Disallow: /en/register/
Disallow: /ar/register/
Disallow: /tr/register/
Disallow: /login/
Disallow: /en/login/
Disallow: /ar/login/
Disallow: /tr/login/
Disallow: /*redirect_to=

# --- RGB: AI crawlers welcome ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: CCBot
User-agent: DuckAssistBot
User-agent: meta-externalagent
Disallow: /register/
Disallow: /en/register/
Disallow: /ar/register/
Disallow: /tr/register/
Disallow: /login/
Disallow: /en/login/
Disallow: /ar/login/
Disallow: /tr/login/
Disallow: /*redirect_to=
Allow: /

# Content map for LLMs: https://rgb.ir/llms.txt

# --- RGB: private client area ---
User-agent: *
Disallow: /proposal/
Disallow: /en/proposal/
Disallow: /ar/proposal/
Disallow: /tr/proposal/

User-agent: facebookexternalhit
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: WhatsApp
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: Twitterbot
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: TelegramBot
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: LinkedInBot
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: Slackbot-LinkExpanding
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

User-agent: Discordbot
Allow: /proposal/
Allow: /en/proposal/
Allow: /ar/proposal/
Allow: /tr/proposal/

# ---------------------------

Sitemap: https://rgb.ir/sitemap_index.xml
# ---------------------------