# Robots.txt for propnex.space
# Generated on January 24, 2035

User-agent: *
Allow: /

# Allow search engines to crawl all important files
Allow: /index.html
Allow: /style.css
Allow: /script.js
Allow: /sitemap.xml
Allow: /404.html

Disallow: /admin/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /*.env$

# Crawl delay for respectful crawling
Crawl-delay: 1

# Sitemap location
Sitemap: https://www.propnex.space/sitemap.xml

# Additional instructions for specific bots
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Disallow AI/ML training scrapers
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Allow images to be indexed
User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot-Image
Allow: /

# Disallow AI scrapers from accessing sensitive data
User-agent: *
Disallow: /*.json$
Disallow: /api/
Disallow: /data/
Disallow: /config/

# Disallow crawling of temporary or system files
Disallow: /*.tmp$
Disallow: /*.log$
Disallow: /private/
Disallow: /search?q=*
Disallow: /*?sort=*