User-agent: *
Sitemap: https://www.egipto.com/sitemap_index.xml

# Disallow all directories and files within
User-agent: *
Disallow: /wp-login.php
Disallow: /wp-admin
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes
Disallow: /cgi-bin
Disallow: /wp-content/plugins/ 
Disallow: /wp-content/themes/ 
Disallow: /wp-includes/ 
Disallow: /*/attachment/
Disallow: /tag/*/page/
Disallow: /tag/*/feed/
Disallow: /comments/
Disallow: /xmlrpc.php
Disallow: /?attachment_id*
Disallow: /?wordfence_lh*
Disallow: /&wordfence_lh*
Disallow: /?hid*
Disallow: /&hid*
Disallow: /package

# Disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.inc$
 
# Disallow parsing individual post feeds, categories and trackbacks..
Disallow: /*trackback*
Disallow: */feed/
Disallow: */feed$

# Search Block
Disallow: /?s= 
Disallow: /search

# Delay
User-agent: noxtrumbot
Crawl-delay: 20
User-agent: Slurp
Crawl-delay: 20

# Spam or useless
User-agent: MSIECrawler
Disallow: / 
User-agent: WebCopier 
Disallow: / 
User-agent: HTTrack 
Disallow: / 
User-agent: Microsoft.URL.Control 
Disallow: / 
User-agent: libwww 
Disallow: / 
User-agent: Orthogaffe 
Disallow: / 
User-agent: UbiCrawler 
Disallow: / 
User-agent: DOC 
Disallow: / 
User-agent: Zao 
Disallow: / 
User-agent: sitecheck.internetseer.com 
Disallow: / 
User-agent: Zealbot 
Disallow: / 
User-agent: MSIECrawler 
Disallow: / 
User-agent: SiteSnagger 
Disallow: / 
User-agent: WebStripper 
Disallow: / 
User-agent: WebCopier 
Disallow: / 
User-agent: Fetch 
Disallow: / 
User-agent: Offline Explorer 
Disallow: / 
User-agent: Teleport 
Disallow: / 
User-agent: TeleportPro 
Disallow: / 
User-agent: WebZIP 
Disallow: / 
User-agent: linko 
Disallow: / 
User-agent: HTTrack 
Disallow: / 
User-agent: Microsoft.URL.Control 
Disallow: / 
User-agent: Xenu 
Disallow: / 
User-agent: larbin 
Disallow: / 
User-agent: libwww 
Disallow: / 
User-agent: ZyBORG 
Disallow: / 
User-agent: Download Ninja 
Disallow: / 
User-agent: wget 
Disallow: / 
User-agent: grub-client 
Disallow: / 
User-agent: k2spider 
Disallow: / 
User-agent: NPBot 
Disallow: / 
User-agent: WebReaper 
Disallow: /

# AI & others
User-agent: GPTBot
Allow: /
User-Agent: PerplexityBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: sentibot
Disallow: /
User-agent: PiplBot
Disallow: /
User-agent: claritybot
Disallow: /
User-agent: Amazonbot
Disallow: /