# =====================================================================
# AARoads Robots.txt
# Purpose: Control crawling of the site, including wiki, forum, images, 
# archives, and shields. Provide Googlebot full access to forum for 
# structured data (DiscussionForumPosting) parsing.
# =====================================================================

# ---------------------------------------------------------------------
# Global rules for all bots
# ---------------------------------------------------------------------
User-agent: *
# Allow everything by default
Allow: /
# Explicitly allow the wiki directory to ensure it's crawled
Allow: /w/

# Sitemap declarations
Sitemap: https://www.aaroads.com/sitemap.xml
Sitemap: https://www.aaroads.com/forum/sitemaps/sitemap-forum-index.xml
Sitemap: https://www.aaroads.com/sitemaps/image-sitemap-index.xml

# Disallow specific non-image paths
Disallow: /omega/guides/
Disallow: /guides/guide.php
Disallow: /guide.php

# Disallow image subdirectories (two-letter codes, e.g., /ca/, /tx/)
# 'Allow: /w/' above overrides wiki directory
Disallow: /[a-z]{2}/

# Disallow archives for two-letter directories
Disallow: /archives/[a-z]{2}/

# Disallow shield images for two-letter directories
Disallow: /shields/img/[a-z]{2}/

# ---------------------------------------------------------------------
# Forum-specific rules for Google crawlers (structured data and content)
# ---------------------------------------------------------------------
User-agent: Googlebot
# Moderate crawl rate for heavy forum content
Crawl-delay: 10
# Prevent crawling of non-essential files
Disallow: /forum/avatars/
Disallow: /forum/cache/
Disallow: /forum/attachments/
# Allow access to dynamic files for SMF forum structured data
Allow: /*.js$
Allow: /*.json$
Allow: /*.php$
# Ensure entire forum content is accessible
Allow: /forum/

# Googlebot image crawler
User-agent: Googlebot-Image
Crawl-delay: 10
Allow: /forum/
Disallow: /forum/avatars/
Disallow: /forum/cache/
Disallow: /forum/attachments/

# Googlebot mobile crawler
User-agent: Googlebot-Mobile
Crawl-delay: 10
Allow: /forum/
Disallow: /forum/avatars/
Disallow: /forum/cache/
Disallow: /forum/attachments/

# AdsBot-Google crawler (for AdSense or related)
User-agent: AdsBot-Google
Crawl-delay: 10
Allow: /forum/

# ---------------------------------------------------------------------
# Forum rules for Bingbot (optional)
# ---------------------------------------------------------------------
User-agent: bingbot
Crawl-delay: 10
Disallow: /forum/avatars/
Disallow: /forum/cache/
Disallow: /forum/attachments/

# ---------------------------------------------------------------------
# All other bots - aggressive restrictions for forum
# ---------------------------------------------------------------------
User-agent: *
Crawl-delay: 10
# Prevent indexing of forum query parameters and user actions
Disallow: /forum/*?sort=
Disallow: /forum/*;prev_next=
Disallow: /forum/*action=post
Disallow: /forum/*action=login
Disallow: /forum/*action=register
# Block unnecessary folders
Disallow: /forum/avatars/
Disallow: /forum/cache/
Disallow: /forum/attachments/