{"schemaVersion":"hugging-bay.ai-crawler-policy.v1","generatedAt":"2026-11-19T07:26:40.110Z","baseUrl":"https://huggingbay.xyz","robotsUrl":"https://huggingbay.xyz/robots.txt","answerEngineManifestUrl":"https://huggingbay.xyz/.well-known/answer-engine.json","botAllowlistUrl":"https://huggingbay.xyz/.well-known/ai-bot-allowlist.json","agentReads":{"summary":"User fetchers and OAI-SearchBot may read find, plan and search APIs. Bulk crawlers use public metadata. Private/export/byte routes stay excluded. Authentication, byte guards and rate limits apply.","userFetchers":["ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Google-GeminiNotebook","MistralAI-User"],"searchCrawlers":["OAI-SearchBot"],"readEndpoints":["https://huggingbay.xyz/api/agents/find-model","https://huggingbay.xyz/api/agents/find-runnable","https://huggingbay.xyz/api/agents/deployment-plan","https://huggingbay.xyz/api/v1/search","https://huggingbay.xyz/api/search","https://huggingbay.xyz/api/search/semantic"],"robots":"https://huggingbay.xyz/robots.txt","authorization":"A user-agent string is not authentication or proof of user consent."},"llmsTxtUrl":"https://huggingbay.xyz/llms.txt","llmsFullTxtUrl":"https://huggingbay.xyz/llms-full.txt","policy":{"publicSearchIndexing":"allowed","publicAiInputForUserAnswers":"allowed","publicMetadataTraining":"allowed for public catalog, public docs, and public hosted-file metadata","frontierAiAssistants":"allowed for public pages and user-requested fetches","openSourceAgents":"allowed through User-agent * plus bounded APIs, sitemaps, llms.txt, and indexing URL batches","privateData":"not exposed; protected by authentication and never listed in public agent manifests","gatedOrAdminEndpoints":"Downloadable doesn't mean tested: this record says whether the exact file can be downloaded now, not that the model is complete or that it runs.","noGatedPrivateBypass":true},"preferredCrawlEntrypoints":["https://huggingbay.xyz/robots.txt","https://huggingbay.xyz/robots.json","https://huggingbay.xyz/.well-known/agent-discovery.json","https://huggingbay.xyz/api/agent-discovery","https://huggingbay.xyz/agents.txt","https://huggingbay.xyz/AGENTS.md","https://huggingbay.xyz/.well-known/grok.json","https://huggingbay.xyz/llms.txt","https://huggingbay.xyz/.well-known/llms.txt","https://huggingbay.xyz/sitemap.xml","https://huggingbay.xyz/api/ai-search","https://huggingbay.xyz/.well-known/ai-search.json","https://huggingbay.xyz/api/agent-recommendation-kit","https://huggingbay.xyz/agent-recommendation-kit","https://huggingbay.xyz/api/citations","https://huggingbay.xyz/.well-known/ai-citations.json","https://huggingbay.xyz/api/ai-visibility","https://huggingbay.xyz/api/traffic","https://huggingbay.xyz/api/ai-mention-monitor","https://huggingbay.xyz/api/ai-mention-results","https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","https://huggingbay.xyz/publishers/launch-kit","https://huggingbay.xyz/api/publishers/launch-kit","https://huggingbay.xyz/api/indexing/status","https://huggingbay.xyz/api/answers","https://huggingbay.xyz/data/manifest.json","https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","https://huggingbay.xyz/feeds/recent.xml","https://huggingbay.xyz/feeds/trending.xml","https://huggingbay.xyz/feeds/hosted-mirrors.xml","https://huggingbay.xyz/api/indexing/urls?kind=answer&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=artifact&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-section&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-evidence&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=publisher&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=downloadable&limit=1000&cursor=0","https://huggingbay.xyz/api/mcp","https://huggingbay.xyz/mcp","https://huggingbay.xyz/browser-extension","https://huggingbay.xyz/browser-extension/manifest.json","https://huggingbay.xyz/api/radar","https://huggingbay.xyz/api/trending?window=7d&limit=25","https://huggingbay.xyz/api/mirror-demand?requestedOnly=1&limit=100","https://huggingbay.xyz/api/mirror-readiness?limit=100"],"preferredCitationTargets":["https://huggingbay.xyz/ai-search","https://huggingbay.xyz/ai-citations","https://huggingbay.xyz/answers","https://huggingbay.xyz/answers/{slug}","https://huggingbay.xyz/artifact/{artifactId}","https://huggingbay.xyz/rankings/top-open-models","https://huggingbay.xyz/rankings/top-open-source-llms","https://huggingbay.xyz/rankings/top-seeded-models","https://huggingbay.xyz/rankings/hosted-run-ready","https://huggingbay.xyz/rankings/hosted-small-local-model-files","https://huggingbay.xyz/rankings/community-verified","https://huggingbay.xyz/rankings/needs-first-review","https://huggingbay.xyz/rankings/new-open-weight-models","https://huggingbay.xyz/rankings/coding-models","https://huggingbay.xyz/rankings/consumer-gpu-models","https://huggingbay.xyz/rankings/preservation-watchlist","https://huggingbay.xyz/rankings/best-7b-8b-models","https://huggingbay.xyz/rankings/best-14b-models","https://huggingbay.xyz/rankings/best-32b-models","https://huggingbay.xyz/rankings/best-70b-models","https://huggingbay.xyz/rankings/global-model-labs","https://huggingbay.xyz/rankings/eastern-open-models","https://huggingbay.xyz/downloads","https://huggingbay.xyz/sources","https://huggingbay.xyz/stacks","https://huggingbay.xyz/publishers/launch-kit"],"userAgents":[{"userAgent":"Google-GeminiNotebook","purpose":"Google user-directed project source fetches","officialDocs":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Google-Agent","purpose":"Google user-directed agent fetches","officialDocs":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GoogleOther","purpose":"Google automatic public-product crawling; not a documented user-triggered fetcher.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages.","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic.","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-webindexer","purpose":"Meta AI search indexing; lets Meta AI cite and link public pages.","allow":["/"],"officialDocs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-externalagent","purpose":"Meta crawler for AI model training and direct product indexing.","allow":["/"],"officialDocs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-externalfetcher","purpose":"Meta user-requested page fetches for Meta AI.","allow":["/"],"officialDocs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"facebookexternalhit","purpose":"Meta link previews when a page is shared in Facebook, Instagram or Messenger.","allow":["/"],"officialDocs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping.","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling.","disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Baiduspider","purpose":"Baidu search and AI answer indexing (China)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Sogou web spider","purpose":"Sogou search indexing (China, powers WeChat search)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Bytespider","purpose":"ByteDance crawling for Toutiao/Douyin search and Doubao AI","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/","/api/artifacts/*/download-plan","/api/v1/artifacts/*/download-plan","/api/downloadable-files","/api/downloadable$","/api/downloadable?","/api/v1/mirrors/hosted","/data/downloadable-files-latest.ndjson","/data/downloadable-files-latest.csv","/hf/*/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ByteDance-Doubao","purpose":"Doubao AI assistant retrieval (ByteDance)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"PetalBot","purpose":"Huawei Petal search and assistant indexing","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YisouSpider","purpose":"Yisou/Shenma search indexing (Alibaba UC)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"360Spider","purpose":"Qihoo 360 search indexing (China)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YandexBot","purpose":"Yandex search and AI answer indexing","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Yeti","purpose":"Naver search indexing (Korea)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"HyperCLOVA","purpose":"Naver HyperCLOVA AI retrieval (Korea)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"coccocbot-web","purpose":"Coc Coc search indexing (Vietnam)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Qwen","purpose":"Alibaba Qwen/Tongyi assistant retrieval","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"KimiBot","purpose":"Moonshot Kimi assistant retrieval (China)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"GLM-Spider","purpose":"Zhipu GLM assistant retrieval (China)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"DeepSeekBot","purpose":"DeepSeek assistant retrieval (China)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"MistralAI-User","purpose":"Mistral user-requested page fetches","officialDocs":"https://docs.mistral.ai/robots","allow":["/","/api/agents/find-model$","/api/agents/find-model?","/api/agents/find-runnable$","/api/agents/find-runnable?","/api/agents/deployment-plan$","/api/agents/deployment-plan?","/api/v1/search$","/api/v1/search?","/api/search$","/api/search?","/api/search/semantic$","/api/search/semantic?"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Amazonbot","purpose":"Amazon Alexa and Rufus answer indexing","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/","/api/artifacts/*/download-plan","/api/v1/artifacts/*/download-plan","/api/downloadable-files","/api/downloadable$","/api/downloadable?","/api/v1/mirrors/hosted","/data/downloadable-files-latest.ndjson","/data/downloadable-files-latest.csv","/hf/*/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Applebot-Extended","purpose":"Apple Intelligence training opt-in (inherits Applebot rules)","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"cohere-ai","purpose":"Cohere user-requested retrieval","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"]},{"userAgent":"Timpibot","purpose":"Timpi decentralized search indexing","allow":["/"],"disallow":["/admin","/auth","/login","/logout","/account","/settings","/publisher/","/publisher$","/library","/saved-kit","/oauth/","/api/admin","/api/auth","/api/account","/api/publisher/","/api/upload","/api/library","/api/me$","/api/me/","/api/watchlist","/api/notifications","/api/sessions","/api/tokens","/api/internal","/api/ops/","/api/identity/","/api/identity$","/api/events/","/api/events$","/api/health","/api/readiness","/api/agents/find-model","/api/agents/find-runnable","/api/agents/deployment-plan","/api/v1/search","/api/search","/api/artifacts?","/api/search/semantic","/api/downloads/","/api/download-completions/","/api/dumps/","/dl/","/api/sandbox/","/api/v1/bakeoff/","/bakeoff/","/downloads/*/","/api/artifacts/*/download-all","/run/api/model","/hf/*/resolve/","/*/*/resolve/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}],"rateGuidance":{"useCursors":true,"maxPublicBatchSize":1000,"preferredCacheSeconds":300,"contact":"Use an identifying User-Agent with a product or contact URL.","avoid":["Do not crawl hash routes as separate recorded pages.","Do not request unbounded /api/artifacts without summary=1, limit, and cursor.","Bulk indexing crawlers should not follow /api/downloads/ or /api/download-completions/; use hosted-file metadata and manifests instead.","Do not attempt token-gated admin or private endpoints as crawler surfaces."]},"note":{"summary":"Downloadable doesn't mean tested: this record says whether the exact file can be downloaded now, not that the model is complete or that it runs."}}