# AUSTRALIAN COMMUNITY MEDIA (ACM) POLICY STATEMENT
# Australian Community Media expressly prohibits the use of any ACM
# content or data, including associated metadata, for any machine
# learning and/or artificial intelligence purposes, including the
# training, development, or operation of AI technologies, tools, or
# machine learning models.
# View our terms of use - https://www.armidaleexpress.com.au/conditions-of-use/

# ============================================================
# robots.txt
# Last updated: April 2026
# ============================================================

# ------------------------------------------------------------
# Sitemaps
# ------------------------------------------------------------
Sitemap: https://www.armidaleexpress.com.au/sitemap.xml
Sitemap: https://www.armidaleexpress.com.au/sitemap-news.xml
Sitemap: https://www.armidaleexpress.com.au/sitemap-sections.xml
Sitemap: https://www.armidaleexpress.com.au/sitemap-topics.xml
Sitemap: https://www.armidaleexpress.com.au/sitemap-explore.xml
Sitemap: https://www.armidaleexpress.com.au/sitemap-2026.xml

# ------------------------------------------------------------
# All crawlers: general rules
# ------------------------------------------------------------
User-agent: *
Disallow: /*/ajax/
Disallow: /assets/
Disallow: /internal/
Disallow: /push-worker.js
Disallow: /scores-and-draws/
Disallow: /*/rss.xml
Disallow: /*/rss-full.xml
Disallow: /*/feed
Disallow: /*/feed/

# ------------------------------------------------------------
# Low-quality / archive / scraper bots
# ------------------------------------------------------------

User-agent: PetalBot
Disallow: /

User-agent: 80legs
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: NeevaBot
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: archive.today
Disallow: /

User-agent: archive.li
Disallow: /

User-agent: archive.is
Disallow: /

User-agent: archive.ph
Disallow: /

User-agent: archive.md
Disallow: /

User-agent: removepaywalls
Disallow: /

User-agent: PrintFriendly
Disallow: /

# ------------------------------------------------------------
# AI crawlers: accessing paywalled content
# ------------------------------------------------------------

# Meta crawlers: explicitly allowed

User-agent: facebookexternalhit
Allow: /
 
User-agent: Meta-WebIndexer
Allow: /
 
User-agent: meta-webindexer
Allow: /
 
User-agent: Meta-ExternalAds
Allow: /
 
User-agent: meta-externalads
Allow: /
 
User-agent: Meta-ExternalAgent
Allow: /
 
User-agent: meta-externalagent
Allow: /
 
User-agent: Meta-ExternalFetcher
Allow: /
 
User-agent: meta-externalfetcher
Allow: /               

# OpenAI
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

# Google AI
User-agent: Google-Extended
Disallow: /

# Anthropic
User-agent: ClaudeBot
Disallow: /

User-agent: claude-web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Amzn-User
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

# Mantis
User-agent: Mantisbot/1.0
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Timpibot (Mistral)
User-agent: Timpibot
Disallow: /

# YouBot (You.com)
User-agent: YouBot
Disallow: /