# Robots.txt for nemahbites.com
# https://www.robotstxt.org/

# Content Signals (https://contentsignals.org/) — machine-readable preferences
# for how collected content may be used:
#   search:   building a search index and showing links/snippets (citations)
#   ai-input: using content as input to AI answers, with attribution
#   ai-train: using content to train or fine-tune AI models
# We allow all three: we want AI assistants and answer engines to know our
# catalog and cite nemahbites.com.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# Agent surfaces — fetchable so agents can use them, noindex so they never rank.
Allow: /api/commerce/config
Allow: /api/catalogue-summary
Allow: /api/agent/v1
Allow: /api/agent/v1/products
Allow: /api/agent/v1/products/
Allow: /api/agent/v1/capabilities
Allow: /api/agent/v1/whoami
Allow: /api/agent/v1/catalog
Allow: /api/agent/v1/categories
Allow: /api/agent/v1/brands
Allow: /api/agent/v1/openapi.json
Allow: /api/agent/identity
Allow: /api/mcp
Allow: /api/a2a
Allow: /api/oauth/
Allow: /.well-known/
Allow: /ai/products/feed.json
Allow: /openapi.json
Allow: /auth.md

Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /checkout
Disallow: /cart
Disallow: /login
Disallow: /register
# Internal search and faceted navigation — an unbounded URL surface.
Disallow: /search
Disallow: /*?*sort=
Disallow: /*?*page=
Disallow: /*?*min_price=
Disallow: /*?*max_price=
Disallow: /*?*brand=
Disallow: /*?*cat=
Disallow: /*?*fv=

User-agent: Googlebot
Allow: /
# Agent surfaces — fetchable so agents can use them, noindex so they never rank.
Allow: /api/commerce/config
Allow: /api/catalogue-summary
Allow: /api/agent/v1
Allow: /api/agent/v1/products
Allow: /api/agent/v1/products/
Allow: /api/agent/v1/capabilities
Allow: /api/agent/v1/whoami
Allow: /api/agent/v1/catalog
Allow: /api/agent/v1/categories
Allow: /api/agent/v1/brands
Allow: /api/agent/v1/openapi.json
Allow: /api/agent/identity
Allow: /api/mcp
Allow: /api/a2a
Allow: /api/oauth/
Allow: /.well-known/
Allow: /ai/products/feed.json
Allow: /openapi.json
Allow: /auth.md
Disallow: /admin/
Disallow: /api/
# Internal search and faceted navigation — an unbounded URL surface.
Disallow: /search
Disallow: /*?*sort=
Disallow: /*?*page=
Disallow: /*?*min_price=
Disallow: /*?*max_price=
Disallow: /*?*brand=
Disallow: /*?*cat=
Disallow: /*?*fv=

User-agent: Bingbot
Allow: /
# Agent surfaces — fetchable so agents can use them, noindex so they never rank.
Allow: /api/commerce/config
Allow: /api/catalogue-summary
Allow: /api/agent/v1
Allow: /api/agent/v1/products
Allow: /api/agent/v1/products/
Allow: /api/agent/v1/capabilities
Allow: /api/agent/v1/whoami
Allow: /api/agent/v1/catalog
Allow: /api/agent/v1/categories
Allow: /api/agent/v1/brands
Allow: /api/agent/v1/openapi.json
Allow: /api/agent/identity
Allow: /api/mcp
Allow: /api/a2a
Allow: /api/oauth/
Allow: /.well-known/
Allow: /ai/products/feed.json
Allow: /openapi.json
Allow: /auth.md
Disallow: /admin/
Disallow: /api/
# Internal search and faceted navigation — an unbounded URL surface.
Disallow: /search
Disallow: /*?*sort=
Disallow: /*?*page=
Disallow: /*?*min_price=
Disallow: /*?*max_price=
Disallow: /*?*brand=
Disallow: /*?*cat=
Disallow: /*?*fv=

User-agent: Googlebot-Image
Allow: /
Disallow: /admin/

# Social preview crawlers (need to render pages for OG previews)
User-agent: facebookexternalhit
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# AI assistants, answer engines & their crawlers — welcome on public content.
# We want the catalog discoverable and citable by AI; only private/transactional
# surfaces stay off limits. See also: https://nemahbites.com/llms.txt and https://nemahbites.com/developers
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: meta-externalagent
Allow: /
# Agent surfaces — fetchable so agents can use them, noindex so they never rank.
Allow: /api/commerce/config
Allow: /api/catalogue-summary
Allow: /api/agent/v1
Allow: /api/agent/v1/products
Allow: /api/agent/v1/products/
Allow: /api/agent/v1/capabilities
Allow: /api/agent/v1/whoami
Allow: /api/agent/v1/catalog
Allow: /api/agent/v1/categories
Allow: /api/agent/v1/brands
Allow: /api/agent/v1/openapi.json
Allow: /api/agent/identity
Allow: /api/mcp
Allow: /api/a2a
Allow: /api/oauth/
Allow: /.well-known/
Allow: /ai/products/feed.json
Allow: /openapi.json
Allow: /auth.md
Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /checkout
Disallow: /cart
Disallow: /login
Disallow: /register
# Internal search and faceted navigation — an unbounded URL surface.
Disallow: /search
Disallow: /*?*sort=
Disallow: /*?*page=
Disallow: /*?*min_price=
Disallow: /*?*max_price=
Disallow: /*?*brand=
Disallow: /*?*cat=
Disallow: /*?*fv=

# Bytespider ignores rate limits and offers no assistant/citation surface
User-agent: Bytespider
Disallow: /

Sitemap: https://nemahbites.com/sitemap.xml