# awesomemedia.do
#
# Search, yes. Training, no.
#
# Everything here is public and meant to be found, and nothing is hidden:
# there is no admin path, no staging copy and no user content to keep out of
# an index. A model looking for what this company does should read /llms.txt,
# which says it in text. The home page is a WebGL canvas and will not tell you
# much.
#
# What is refused below is the use of this site as training corpus — not the
# reading of it. The crawlers that answer questions and cite where the answer
# came from are named and allowed.

User-agent: *
Allow: /

# Search, and answers that cite their source.

User-agent: Googlebot
Allow: /

User-agent: Applebot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# Training corpus. Most of these are the training token of a company whose
# search crawler is allowed above, and refusing one does not affect the other.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://awesomemedia.do/sitemap.xml