# Robots.txt - Optimized for Search Engines (SEO) & AI Search Engines (GEO/AIO)

# SECTION 1: Unlocked AI Search Engines to drive traffic (Generative Engine Optimization)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: YouBot
Allow: /

# SECTION 2: Block pure AI Training Scrapers and Data Brokers (No traffic value)
User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: OmgiliBot
Disallow: /

User-agent: diffbot
Disallow: /

# SECTION 3: Block aggressive SEO audit crawlers that waste server resources and bandwidth
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Rogerbot
Disallow: /

# SECTION 4: Guidelines for general search engine crawlers (Google, Bing, Yandex, etc.)
User-agent: *
# Disallow private/admin systems and directories
Disallow: /admincp/
Disallow: /admin-panel/
Disallow: /api/
Disallow: /api.php
Disallow: /xhr/
Disallow: /script_backups/
Disallow: /updates/
Disallow: /install/
Disallow: /cache/
Disallow: /nodejs/
Disallow: /xml/
Disallow: /ajax_loading.php
Disallow: /system_status.php
Disallow: /logout
Disallow: /messages/
Disallow: /setting/
Disallow: /page-setting/
Disallow: /group-setting/

# Disallow system folders but allow static assets (Images, CSS, JS) for indexing and page experience
Disallow: /assets/
Allow: /assets/img/
Allow: /assets/css/
Allow: /assets/js/
Allow: /assets/fonts/

Disallow: /sources/
Disallow: /themes/
Allow: /themes/*/layout/
Allow: /themes/*/stylesheet/
Allow: /themes/*/javascript/
Allow: /themes/*/img/

# Allow crawling of sitemap folder
Allow: /sitemap/

Sitemap: https://vlike.vn/sitemap/sitemap-index.xml