# ============================================================
# robots.txt for www.lyrics.my
# Last updated: April 2026
# ============================================================

# ============================================================
# SECTION 1: LEGITIMATE SEARCH ENGINE CRAWLERS
# Explicitly allow major search engines for full indexing.
# ============================================================

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: facebot
Allow: /

User-agent: ia_archiver
Allow: /

# ============================================================
# SECTION 2: BLOCK AI TRAINING & SCRAPING BOTS
# These bots harvest content for AI model training without
# providing traffic in return. All blocked from entire site.
# ============================================================

# --- OpenAI ---
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

# --- Anthropic ---
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

# --- Google AI (separate from search indexing) ---
User-agent: Google-Extended
Disallow: /

# --- Apple ---
User-agent: Applebot-Extended
Disallow: /

# --- Amazon ---
User-agent: Amazonbot
Disallow: /

# --- Meta / Facebook ---
User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

# --- Bytedance (TikTok) ---
User-agent: Bytespider
Disallow: /

# --- Common Crawl (used widely for AI training datasets) ---
User-agent: CCBot
Disallow: /

# --- Cohere ---
User-agent: cohere-ai
Disallow: /

# --- Perplexity ---
User-agent: PerplexityBot
Disallow: /

# --- Diffbot ---
User-agent: Diffbot
Disallow: /

# --- ImagesiftBot ---
User-agent: ImagesiftBot
Disallow: /

# --- Omgili / Webz.io ---
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# --- YouBot (You.com AI) ---
User-agent: YouBot
Disallow: /

# --- Timpibot (Timpi search AI) ---
User-agent: Timpibot
Disallow: /

# --- DataForSeoBot ---
User-agent: DataForSeoBot
Disallow: /

# --- PetalBot (Huawei AI) ---
User-agent: PetalBot
Disallow: /

# --- Scrapy (common scraping framework) ---
User-agent: Scrapy
Disallow: /

# ============================================================
# SECTION 3: GENERAL RULES FOR ALL OTHER BOTS
# Allow crawling of public content but block low-value,
# duplicate, and sensitive paths.
# ============================================================

User-agent: *

# -- Block sensitive/admin areas --
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /user/
Disallow: /account/
Disallow: /dashboard/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /cgi-bin/

# -- Block internal search results (infinite URL space, low value) --
Disallow: /search
Disallow: /search/
Disallow: /?s=
Disallow: /?q=
Disallow: /*?s=
Disallow: /*?q=

# -- Block share widget and utility paths --
Disallow: /share-widget
Disallow: /share-widget/

# -- Block API endpoints --
Disallow: /api/
Disallow: /api-internal/

# -- Block duplicate/low-value paginated tag archives --
Disallow: /tag/
Disallow: /tags/

# -- Allow everything else --
Allow: /

# ============================================================
# SITEMAP
# ============================================================

Sitemap: https://www.lyrics.my/sitemap.xml