# robots.txt — varietyengines.com
# Reviewed 2026-09-03.
#
# POLICY, so it is not undone by accident:
# Crawlers that can send this site referral traffic or cite it in an answer are
# ALLOWED. That deliberately includes AI answer engines — they are a discovery
# channel, not a threat. Only bulk crawlers with no discovery value are blocked.
# Do not add a blanket AI block here without reading the notes below.

# ---------------------------------------------------------------------------
# Default policy — everything not named below
# ---------------------------------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Crawl-budget traps (cart actions, faceted nav, sorting, on-site search)
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?rating_filter=
Disallow: /*?s=
Disallow: /*?wc-ajax=
Disallow: /*?p=

# WooCommerce system files
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# ---------------------------------------------------------------------------
# ANSWER ENGINES / AI SEARCH — allowed.
# These fetch a page to answer a live user question and link back to the
# source. Blocking them removes the site from the answer, not from the model.
#   OAI-SearchBot, ChatGPT-User  — ChatGPT Search
#   PerplexityBot, Perplexity-User — Perplexity
#   Claude-SearchBot, Claude-User  — Claude web search
#   DuckAssistBot                  — DuckDuckGo AI assist
#   Applebot                       — Siri / Spotlight / Apple search
#   Amazonbot                      — Alexa / Amazon shopping answers
# ---------------------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: Applebot
User-agent: Amazonbot
Allow: /
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-track/
Disallow: /wishlist/
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?rating_filter=
Disallow: /*?s=
Disallow: /*?wc-ajax=
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# ---------------------------------------------------------------------------
# MODEL / GROUNDING CRAWLERS — also allowed, deliberately.
#   Google-Extended     — NOT a crawler. It is the switch that decides whether
#                         Google may use already-crawled pages in AI Overviews
#                         and Gemini grounding. Disallowing it removes this site
#                         from AI Overviews. Leave allowed.
#   Applebot-Extended   — Apple Intelligence. Separate from Applebot above.
#   GPTBot, ClaudeBot   — build the model's background knowledge of the brand.
#   meta-externalagent  — Meta AI.
#   CCBot               — Common Crawl; feeds many downstream search/AI tools.
# A store this size has no licensing leverage to gain by blocking these, and
# the line between "training" and "grounding" keeps moving. Allowing costs
# nothing; blocking risks silently dropping out of a citation surface.
# ---------------------------------------------------------------------------
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: meta-externalagent
User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-track/
Disallow: /wishlist/
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?rating_filter=
Disallow: /*?s=
Disallow: /*?wc-ajax=
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# ---------------------------------------------------------------------------
# BLOCKED — crawls hard, returns nothing. No search or referral surface that
# reaches a US freight-shipped auto-parts buyer.
# ---------------------------------------------------------------------------
User-agent: Bytespider
Disallow: /

Sitemap: https://varietyengines.com/sitemap_index.xml