# Hugging Bay is intentionally crawlable for search, answer engines, and user-requested AI fetches.
# Public pages are open. Selected agent reads are allowed; private, bulk and byte paths stay excluded.
# These rules guide crawlers and do not replace authentication.
# Need faster or bulk access? Request access: https://huggingbay.xyz/crawl-access
# Request access by API: POST https://huggingbay.xyz/api/v1/crawl-access/requests (contact_email, intended_use). Summary: https://huggingbay.xyz/.well-known/crawl-access.json
# Bulk-Catalog: the whole catalog in one daily download: https://huggingbay.xyz/data (changes feed: https://huggingbay.xyz/api/v1/catalog/changes?since=YYYY-MM-DD)
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /auth
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /settings
Disallow: /publisher/
Disallow: /publisher$
Disallow: /library
Disallow: /saved-kit
Disallow: /oauth/
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/account
Disallow: /api/publisher/
Disallow: /api/upload
Disallow: /api/library
Disallow: /api/me$
Disallow: /api/me/
Disallow: /api/watchlist
Disallow: /api/notifications
Disallow: /api/sessions
Disallow: /api/tokens
Disallow: /api/internal
Disallow: /api/ops/
Disallow: /api/identity/
Disallow: /api/identity$
Disallow: /api/events/
Disallow: /api/events$
Disallow: /api/health
Disallow: /api/readiness
Disallow: /api/agents/find-model
Disallow: /api/agents/find-runnable
Disallow: /api/agents/deployment-plan
Disallow: /api/v1/search
Disallow: /api/search
Disallow: /api/artifacts?
Disallow: /api/search/semantic
Disallow: /api/downloads/
Disallow: /api/download-completions/
Disallow: /api/dumps/
Disallow: /dl/
Disallow: /api/sandbox/
Disallow: /api/v1/bakeoff/
Disallow: /bakeoff/
Disallow: /downloads/*/
Disallow: /api/artifacts/*/download-all
Disallow: /run/api/model
Disallow: /hf/*/resolve/
Disallow: /*/*/resolve/

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Googlebot
User-agent: CCBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: meta-webindexer
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: facebookexternalhit
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /auth
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /settings
Disallow: /publisher/
Disallow: /publisher$
Disallow: /library
Disallow: /saved-kit
Disallow: /oauth/
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/account
Disallow: /api/publisher/
Disallow: /api/upload
Disallow: /api/library
Disallow: /api/me$
Disallow: /api/me/
Disallow: /api/watchlist
Disallow: /api/notifications
Disallow: /api/sessions
Disallow: /api/tokens
Disallow: /api/internal
Disallow: /api/ops/
Disallow: /api/identity/
Disallow: /api/identity$
Disallow: /api/events/
Disallow: /api/events$
Disallow: /api/health
Disallow: /api/readiness
Disallow: /api/agents/find-model
Disallow: /api/agents/find-runnable
Disallow: /api/agents/deployment-plan
Disallow: /api/v1/search
Disallow: /api/search
Disallow: /api/artifacts?
Disallow: /api/search/semantic
Disallow: /api/downloads/
Disallow: /api/download-completions/
Disallow: /api/dumps/
Disallow: /dl/
Disallow: /api/sandbox/
Disallow: /api/v1/bakeoff/
Disallow: /bakeoff/
Disallow: /downloads/*/
Disallow: /api/artifacts/*/download-all
Disallow: /run/api/model
Disallow: /hf/*/resolve/
Disallow: /*/*/resolve/

User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Google-Agent
User-agent: Google-GeminiNotebook
User-agent: MistralAI-User
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/agents/find-model$
Allow: /api/agents/find-model?
Allow: /api/agents/find-runnable$
Allow: /api/agents/find-runnable?
Allow: /api/agents/deployment-plan$
Allow: /api/agents/deployment-plan?
Allow: /api/v1/search$
Allow: /api/v1/search?
Allow: /api/search$
Allow: /api/search?
Allow: /api/search/semantic$
Allow: /api/search/semantic?
Allow: /auth.md
Disallow: /admin
Disallow: /auth
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /settings
Disallow: /publisher/
Disallow: /publisher$
Disallow: /library
Disallow: /saved-kit
Disallow: /oauth/
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/account
Disallow: /api/publisher/
Disallow: /api/upload
Disallow: /api/library
Disallow: /api/me$
Disallow: /api/me/
Disallow: /api/watchlist
Disallow: /api/notifications
Disallow: /api/sessions
Disallow: /api/tokens
Disallow: /api/internal
Disallow: /api/ops/
Disallow: /api/identity/
Disallow: /api/identity$
Disallow: /api/events/
Disallow: /api/events$
Disallow: /api/health
Disallow: /api/readiness
Disallow: /api/agents/find-model
Disallow: /api/agents/find-runnable
Disallow: /api/agents/deployment-plan
Disallow: /api/v1/search
Disallow: /api/search
Disallow: /api/artifacts?
Disallow: /api/search/semantic
Disallow: /api/downloads/
Disallow: /api/download-completions/
Disallow: /api/dumps/
Disallow: /dl/
Disallow: /api/sandbox/
Disallow: /api/v1/bakeoff/
Disallow: /bakeoff/
Disallow: /downloads/*/
Disallow: /api/artifacts/*/download-all
Disallow: /run/api/model
Disallow: /hf/*/resolve/
Disallow: /*/*/resolve/

User-agent: Bytespider
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /auth
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /settings
Disallow: /publisher/
Disallow: /publisher$
Disallow: /library
Disallow: /saved-kit
Disallow: /oauth/
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/account
Disallow: /api/publisher/
Disallow: /api/upload
Disallow: /api/library
Disallow: /api/me$
Disallow: /api/me/
Disallow: /api/watchlist
Disallow: /api/notifications
Disallow: /api/sessions
Disallow: /api/tokens
Disallow: /api/internal
Disallow: /api/ops/
Disallow: /api/identity/
Disallow: /api/identity$
Disallow: /api/events/
Disallow: /api/events$
Disallow: /api/health
Disallow: /api/readiness
Disallow: /api/agents/find-model
Disallow: /api/agents/find-runnable
Disallow: /api/agents/deployment-plan
Disallow: /api/v1/search
Disallow: /api/search
Disallow: /api/artifacts?
Disallow: /api/search/semantic
Disallow: /api/downloads/
Disallow: /api/download-completions/
Disallow: /api/dumps/
Disallow: /dl/
Disallow: /api/sandbox/
Disallow: /api/v1/bakeoff/
Disallow: /bakeoff/
Disallow: /downloads/*/
Disallow: /api/artifacts/*/download-all
Disallow: /run/api/model
Disallow: /hf/*/resolve/
Disallow: /*/*/resolve/
Disallow: /api/artifacts/*/download-plan
Disallow: /api/v1/artifacts/*/download-plan
Disallow: /api/downloadable-files
Disallow: /api/downloadable$
Disallow: /api/downloadable?
Disallow: /api/v1/mirrors/hosted
Disallow: /data/downloadable-files-latest.ndjson
Disallow: /data/downloadable-files-latest.csv
Disallow: /hf/*/*/resolve/
Disallow: /*/*/resolve/

User-agent: ShapBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /auth
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /settings
Disallow: /publisher/
Disallow: /publisher$
Disallow: /library
Disallow: /saved-kit
Disallow: /oauth/
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/account
Disallow: /api/publisher/
Disallow: /api/upload
Disallow: /api/library
Disallow: /api/me$
Disallow: /api/me/
Disallow: /api/watchlist
Disallow: /api/notifications
Disallow: /api/sessions
Disallow: /api/tokens
Disallow: /api/internal
Disallow: /api/ops/
Disallow: /api/identity/
Disallow: /api/identity$
Disallow: /api/events/
Disallow: /api/events$
Disallow: /api/health
Disallow: /api/readiness
Disallow: /api/agents/find-model
Disallow: /api/agents/find-runnable
Disallow: /api/agents/deployment-plan
Disallow: /api/v1/search
Disallow: /api/search
Disallow: /api/artifacts?
Disallow: /api/search/semantic
Disallow: /api/downloads/
Disallow: /api/download-completions/
Disallow: /api/dumps/
Disallow: /dl/
Disallow: /api/sandbox/
Disallow: /api/v1/bakeoff/
Disallow: /bakeoff/
Disallow: /downloads/*/
Disallow: /api/artifacts/*/download-all
Disallow: /run/api/model
Disallow: /hf/*/resolve/
Disallow: /*/*/resolve/
Crawl-delay: 10

Sitemap: https://huggingbay.xyz/sitemap.xml
# Robots-JSON: https://huggingbay.xyz/robots.json
# Agents: https://huggingbay.xyz/agents.txt
# LLMs: https://huggingbay.xyz/llms.txt
# LLMs-zh: https://huggingbay.xyz/llms-zh.txt
# LLMs-Well-Known: https://huggingbay.xyz/.well-known/llms.txt
# LLMs-Full: https://huggingbay.xyz/llms-full.txt
# Content-Signal: search = building a search index and showing results; ai-input = using content as input to AI answers; ai-train = training or fine-tuning AI models.
# API-Catalog: https://huggingbay.xyz/.well-known/api-catalog
# Agent-Skills: https://huggingbay.xyz/.well-known/agent-skills/index.json
# Auth: https://huggingbay.xyz/auth.md
# Agent-Discovery: https://huggingbay.xyz/.well-known/agent-discovery.json
# Agent-Discovery-API: https://huggingbay.xyz/api/agent-discovery
# AI-Search: https://huggingbay.xyz/api/ai-search
# AI-Search-Well-Known: https://huggingbay.xyz/.well-known/ai-search.json
# Agent-Recommendation-Kit: https://huggingbay.xyz/api/agent-recommendation-kit
# AI-Citations: https://huggingbay.xyz/api/citations
# AI-Citations-Well-Known: https://huggingbay.xyz/.well-known/ai-citations.json
# AI-Visibility: https://huggingbay.xyz/api/ai-visibility
# External-Visibility: https://huggingbay.xyz/external-visibility
# External-Visibility-API: https://huggingbay.xyz/api/external-visibility
# AI-Mention-Monitor: https://huggingbay.xyz/api/ai-mention-monitor
# AI-Mention-Results: https://huggingbay.xyz/api/ai-mention-results
# Public-Traffic: https://huggingbay.xyz/api/traffic
# Feed: https://huggingbay.xyz/feed.xml
# Recent-Feed: https://huggingbay.xyz/feeds/recent.xml
# Trending-Feed: https://huggingbay.xyz/feeds/trending.xml
# Hosted-Mirrors-Feed: https://huggingbay.xyz/feeds/hosted-mirrors.xml
# Data-Manifest: https://huggingbay.xyz/data/manifest.json
# Catalog-Dump: https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000
# Hosted-Files-Dump: https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000
# Agent-Manifest: https://huggingbay.xyz/.well-known/ai-catalog.json
# Answer-Engine: https://huggingbay.xyz/.well-known/answer-engine.json
# Answer-Packs: https://huggingbay.xyz/api/answers
# AI-Crawler-Policy: https://huggingbay.xyz/.well-known/ai-crawler-policy.json
# AI-Bot-Allowlist: https://huggingbay.xyz/.well-known/ai-bot-allowlist.json
# IndexNow: https://huggingbay.xyz/api/indexing/indexnow
# IndexNow-Key: https://huggingbay.xyz/96cf908e785257c158c7ae6d031da825.txt
# MCP: https://huggingbay.xyz/.well-known/mcp.json
# Remote-MCP: https://huggingbay.xyz/api/mcp
# Remote-MCP-Alias: https://huggingbay.xyz/mcp
# Crawl-Access: https://huggingbay.xyz/crawl-access
# Crawl-Access-API: https://huggingbay.xyz/api/v1/crawl-access
# Crawl-Access-JSON: https://huggingbay.xyz/.well-known/crawl-access.json