# robots.txt for russ-shilling.com
# Updated: March 25, 2026
# Policy: Allow search engines and AI crawlers, block SEO scrapers

# As a condition of accessing this website, you agree to
# abide by the following content signals:
# (a) If a content-signal = yes, you may collect content
# for the corresponding use.
# (b) If a content-signal = no, you may not collect
# content for the corresponding use.
# (c) If the website operator does not include a content
# signal for a corresponding use, the website operator
# neither grants nor restricts permission via content
# signal with respect to the corresponding use.
#
# The content signals and their meanings are:
# search: building a search index and providing search
# results (e.g., returning hyperlinks and short excerpts
# from your website's contents). Search does not include
# providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models
# (e.g., retrieval augmented generation, grounding, or
# other real-time taking of content for generative AI
# search answers).
# ai-train: training or fine-tuning AI models.
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE
# EXPRESS RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE
# EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND
# RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

# ----------------------------
# SEARCH ENGINES
# ----------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ----------------------------
# AI & LLM CRAWLERS
# ----------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ----------------------------
# SOCIAL MEDIA LINK PREVIEWS
# ----------------------------
User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Discordbot
Allow: /

# ----------------------------
# BLOCKED: SEO scrapers, data brokers, and bandwidth hogs
# ----------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: HubSpot-Crawler
Disallow: /

User-agent: IbouBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SentiBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: 360Spider
Disallow: /

Sitemap: https://russ-shilling.com/sitemap.xml
Sitemap: https://russ-shilling.com/sitemap.xml.gz
Llms-txt: https://russ-shilling.com/llms.txt