# robots.txt — connectww.net # Connect Network Inc. | Integrated Marketing Agency # Updated: May 2026 # ============================================================ # STANDARD SEARCH ENGINES # ============================================================ User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: Yandex Allow: / User-agent: Applebot Allow: / # ============================================================ # AI CRAWLERS — OPENAI / CHATGPT # GPTBot = training data # OAI-SearchBot = real-time search index # ChatGPT-User = user-initiated browsing in ChatGPT # OAI-AdsBot = ChatGPT ads validation (Apr 2026) # ============================================================ User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-AdsBot Allow: / # ============================================================ # AI CRAWLERS — ANTHROPIC / CLAUDE # ClaudeBot = training data # Claude-SearchBot = search indexing # Claude-User = user-initiated browsing in Claude # anthropic-ai = legacy user agent (still active in logs) # ============================================================ User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / User-agent: anthropic-ai Allow: / # ============================================================ # AI CRAWLERS — GOOGLE # Google-Extended = Gemini training (separate from Googlebot) # Google-CloudVertexBot = Vertex AI / internal R&D # GoogleOther = generic Google crawls # Google-Read-Aloud = text-to-speech feature # Google-NotebookLM = NotebookLM AI product # ============================================================ User-agent: Google-Extended Allow: / User-agent: Google-CloudVertexBot Allow: / User-agent: GoogleOther Allow: / User-agent: Google-Read-Aloud Allow: / User-agent: Google-NotebookLM Allow: / # ============================================================ # AI CRAWLERS — PERPLEXITY # PerplexityBot = search indexing # Perplexity-User = user-initiated browsing # ============================================================ User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # ============================================================ # AI CRAWLERS — META / FACEBOOK # meta-externalagent = Meta AI (Llama) training & search # FacebookBot = language model improvement # ============================================================ User-agent: meta-externalagent Allow: / User-agent: FacebookBot Allow: / # ============================================================ # AI CRAWLERS — APPLE # Applebot-Extended = Apple AI / Siri / Spotlight training # (Applebot listed above under Search Engines) # ============================================================ User-agent: Applebot-Extended Allow: / # ============================================================ # AI CRAWLERS — xAI / GROK # ============================================================ User-agent: Grok Allow: / User-agent: GrokBot Allow: / # ============================================================ # AI CRAWLERS — MISTRAL AI # ============================================================ User-agent: MistralAI-User Allow: / # ============================================================ # AI CRAWLERS — AMAZON # Amazonbot = Alexa AI and Rufus shopping assistant # ============================================================ User-agent: Amazonbot Allow: / # ============================================================ # AI CRAWLERS — DEEPSEEK # ============================================================ User-agent: DeepSeekBot Allow: / # ============================================================ # AI CRAWLERS — YOU.COM # ============================================================ User-agent: YouBot Allow: / # ============================================================ # AI CRAWLERS — COHERE # ============================================================ User-agent: cohere-ai Allow: / # ============================================================ # AI CRAWLERS — BYTEDANCE / TIKTOK # Bytespider = powers Doubao and internal LLMs # ============================================================ User-agent: Bytespider Allow: / # ============================================================ # AI CRAWLERS — COMMON CRAWL # CCBot = open dataset used by many AI labs for training # ============================================================ User-agent: CCBot Allow: / # ============================================================ # AI CRAWLERS — ALLEN INSTITUTE FOR AI (AI2) # ai2bot = general research crawler # ai2bot-dolma = Dolma open training dataset # ============================================================ User-agent: ai2bot Allow: / User-agent: ai2bot-dolma Allow: / # ============================================================ # AI CRAWLERS — DUCKDUCKGO AI # DuckAssistBot = DuckDuckGo AI answers # ============================================================ User-agent: DuckAssistBot Allow: / # ============================================================ # AI CRAWLERS — DIFFBOT # ============================================================ User-agent: Diffbot Allow: / # ============================================================ # ALL OTHER BOTS — DEFAULT RULES # ============================================================ User-agent: * # --- ALLOW: Canonical service pages (keyword URLs) --- Allow: /integrated-marketing-agency Allow: /programmatic-advertising-agency Allow: /ooh-advertising-agency Allow: /experiential-marketing-agency Allow: /sports-marketing-agency Allow: /smart-city-solutions Allow: /csr-advisory-firm # --- ALLOW: Group pages --- Allow: /aboutus Allow: /people Allow: /people/leadership Allow: /values Allow: /work Allow: /clients Allow: /immersive # --- ALLOW: Contact and utility pages --- Allow: /news Allow: /office Allow: /careers Allow: /new-business Allow: /privacy-policy # --- ALLOW: AI guidance files --- Allow: /llms.txt Allow: /llms-full.txt # --- DISALLOW: Broken PHP error page (server path exposed) --- Disallow: /home # --- DISALLOW: Duplicate service URLs (Base64 ID system) --- # These are live duplicates of the keyword service URLs above. # Blocking consolidates crawl authority to canonical URLs. Disallow: /service-details/ # --- DISALLOW: Framework entry point --- Disallow: /index.php Disallow: /send_email Disallow: /invitation_add_db # ============================================================ # SITEMAP # ============================================================ Sitemap: https://connectww.net/sitemap.xml