# Baiduspider
## Too much requests and not in target area
User-agent: Baiduspider
Disallow: /

# Disallow jobdigger
## Too much requests and not in target area
User-Agent: JobdiggerSpider
Disallow: /

User-agent: WebCopier
Disallow: /
 
User-agent: Fetch
Disallow: /
 
User-agent: Offline Explorer
Disallow: /
 
User-agent: Teleport
Disallow: /
 
User-agent: TeleportPro
Disallow: /
 
User-agent: WebZIP
Disallow: /
 
User-agent: linko
Disallow: /
 
User-agent: HTTrack
Disallow: /
 
User-agent: Microsoft.URL.Control
Disallow: /
 
User-agent: Xenu
Disallow: /
 
User-agent: larbin
Disallow: /
 
User-agent: libwww
Disallow: /
 
User-agent: ZyBORG
Disallow: /
 
User-agent: Download Ninja
Disallow: /

# Crawlers Setup
User-agent: *
Crawl-delay: 10

User-agent: *
Allow: /*?page=
Allow: /*?p=
Allow: /*html?$
 
Disallow: /index.php/
Disallow: /*?
Disallow: /*?page=*&
Disallow: /*?p=&
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID= 
Sitemap: https://www.bel-bo.be/media/sitemap/sitemap_fr_be_index.xml
Sitemap: https://www.bel-bo.be/media/sitemap/sitemap_nl_be_index.xml