35 lines
508 B
Plaintext
35 lines
508 B
Plaintext
User-agent: sogou
|
|
Disallow: /
|
|
|
|
User-agent: *
|
|
Disallow: /stuff/
|
|
|
|
User-agent: *
|
|
Allow: /stuff/WebQuake/
|
|
|
|
User-agent: *
|
|
Disallow: /temp/
|
|
|
|
User-agent: *
|
|
#Disallow: /webarchive/
|
|
Disallow: /reviews/random_map.php
|
|
Disallow: /reviews/random_recommendations.php
|
|
|
|
#because he is a twat
|
|
User-agent: bingbot
|
|
Disallow: /46
|
|
Disallow: /206
|
|
Disallow: /52
|
|
|
|
User-Agent: MJ12bot
|
|
#Crawl-Delay: 5
|
|
Disallow: /
|
|
|
|
# fuck off
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
User-agent: SemrushBot-SA
|
|
Disallow: /
|