# robots.txt for FIDH Stop This Movie Archive Site
# This is a static HTML archive - all content should be crawlable

User-agent: *
Allow: /

# Allow access to AI instructions
Allow: /AI.txt

# Block sensitive directories and files
Disallow: /.git/
Disallow: /.github/
Disallow: /.htaccess
Disallow: /.htpasswd/
Disallow: /.env
Disallow: /*.env*
Disallow: /*.log$
Disallow: /*.db$
Disallow: /*.sqlite$
Disallow: /node_modules/
Disallow: /vendor/
Disallow: /bower_components/

# Sitemap location
Sitemap: https://stopthismovie.fidh.org/sitemap.xml

# Crawl delay for politeness
Crawl-delay: 5

# Optional: Clean-param for tracking parameters (uncomment if needed)
# Clean-param: utm_source /
# Clean-param: utm_medium /
# Clean-param: utm_campaign /
# Clean-param: gclid /
# Clean-param: fbclid /
# Clean-param: mc_cid /
# Clean-param: mc_eid /