{
  "schemaVersion": "1.0",
  "site": "https://ai-girlfriend-app-2026.com/",
  "updated": "2026-10-07",
  "policy": "Allow public crawling, indexing, retrieval and training; not a third-party copyright license.",
  "robotsUrl": "https://ai-girlfriend-app-2026.com/robots.txt",
  "defaultRule": {
    "agent": "*",
    "allow": "/"
  },
  "crawlers": [
    {
      "agent": "GPTBot",
      "purpose": "Potential foundation-model training crawl",
      "source": "https://developers.openai.com/api/docs/bots",
      "note": "Independent from search permission; operator publishes IP ranges. Allow indicates crawl preference, not an indexing guarantee.",
      "allow": "/"
    },
    {
      "agent": "OAI-SearchBot",
      "purpose": "ChatGPT search discovery",
      "source": "https://developers.openai.com/api/docs/bots",
      "note": "Search choice is independent of GPTBot. Allow and reachable origin help eligibility; results are not guaranteed.",
      "allow": "/"
    },
    {
      "agent": "ChatGPT-User",
      "purpose": "User-requested retrieval",
      "source": "https://developers.openai.com/api/docs/bots",
      "note": "Not automatic crawling or search inclusion control; robots.txt may not apply to these user actions.",
      "allow": "/"
    },
    {
      "agent": "ClaudeBot",
      "purpose": "Potential model-training collection",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Anthropic says its bots honor robots.txt; separate from user retrieval and search.",
      "allow": "/"
    },
    {
      "agent": "Claude-SearchBot",
      "purpose": "Claude search relevance and indexing",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Distinct search permission; disabling can reduce search visibility. No ranking promise from Allow.",
      "allow": "/"
    },
    {
      "agent": "Claude-User",
      "purpose": "User-directed retrieval",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Anthropic states that its user-directed retrieval crawler honors robots.txt.",
      "allow": "/"
    },
    {
      "agent": "Googlebot",
      "purpose": "Google Search crawling",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "note": "Allow permits crawling but does not ensure indexing, ranking or rich results.",
      "allow": "/"
    },
    {
      "agent": "Google-Extended",
      "purpose": "Gemini training and specified grounding controls",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "note": "A robots.txt product token, not a separate HTTP crawler. Does not affect Google Search inclusion or ranking.",
      "allow": "/"
    },
    {
      "agent": "bingbot",
      "purpose": "Bing search crawling",
      "source": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "note": "Official user-agent includes lowercase bingbot; token matching is case-insensitive. Honors REP; Allow is not an indexing promise.",
      "allow": "/"
    },
    {
      "agent": "PerplexityBot",
      "purpose": "Perplexity search surfacing",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "note": "Operator says not foundation-model training; requires reachable content and compatible WAF policy, with no appearance guarantee.",
      "allow": "/"
    },
    {
      "agent": "Perplexity-User",
      "purpose": "User-requested retrieval",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "note": "Not automatic training crawl; generally ignores robots.txt for user-requested fetches.",
      "allow": "/"
    },
    {
      "agent": "Applebot",
      "purpose": "Apple search discovery and content collection",
      "source": "https://support.apple.com/en-us/119829",
      "note": "Used by Spotlight, Siri and Safari. Collection may feed generative features; Extended controls training use separately.",
      "allow": "/"
    },
    {
      "agent": "Applebot-Extended",
      "purpose": "Apple foundation-model training use control",
      "source": "https://support.apple.com/en-us/119829",
      "note": "Does not crawl pages; controls use of Applebot-collected data. Does not affect Search ranking.",
      "allow": "/"
    },
    {
      "agent": "DuckAssistBot",
      "purpose": "DuckDuckGo AI-assisted answer retrieval",
      "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "note": "Not model training. Its permission does not affect ordinary organic search rankings; changes may take 72 hours.",
      "allow": "/"
    },
    {
      "agent": "Amazonbot",
      "purpose": "Amazon product improvement and potential AI training",
      "source": "https://developer.amazon.com/amazonbot",
      "note": "Automated crawls honor REP. Some robots metadata is honored; crawl-delay is unsupported.",
      "allow": "/"
    },
    {
      "agent": "Amzn-SearchBot",
      "purpose": "Amazon search experiences such as Alexa",
      "source": "https://developer.amazon.com/amazonbot",
      "note": "Not generative-model training. Separate user-agent setting; may follow other search-bot rules when unnamed.",
      "allow": "/"
    },
    {
      "agent": "Amzn-User",
      "purpose": "User-requested live Amazon retrieval",
      "source": "https://developer.amazon.com/amazonbot",
      "note": "Not model-training crawl. User-initiated requests may not follow all robots.txt directives.",
      "allow": "/"
    },
    {
      "agent": "CCBot",
      "purpose": "Common Crawl open web archive",
      "source": "https://commoncrawl.org/ccbot",
      "note": "Public crawl repository can support downstream research and training. Operator warns that user-agent strings can be spoofed.",
      "allow": "/"
    },
    {
      "agent": "FacebookExternalHit",
      "purpose": "Shared-link previews in Meta products",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "May bypass robots.txt for security or integrity checks; preview retrieval is distinct from model-training collection.",
      "allow": "/"
    },
    {
      "agent": "Meta-WebIndexer",
      "purpose": "Meta AI search citations and linking",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Documented search indexing crawler; Allow records crawl preference but does not guarantee inclusion or citations.",
      "allow": "/"
    },
    {
      "agent": "Meta-ExternalAds",
      "purpose": "Ads and business-product improvement",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Distinct product-improvement crawler; its purpose differs from AI search and user-requested retrieval.",
      "allow": "/"
    },
    {
      "agent": "Meta-ExternalAgent",
      "purpose": "AI-model training and direct indexing for product improvement",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Separate from user-requested fetching; an Allow group records public-content crawl permission.",
      "allow": "/"
    },
    {
      "agent": "Meta-ExternalFetcher",
      "purpose": "User-requested links and agentic product features",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Meta says it may bypass robots.txt because the fetch was requested by a user.",
      "allow": "/"
    }
  ]
}