Too lazy to comment

2019-12-31 10:12:16 +01:00
parent 3f4271f835
commit 096c4f552f
385 changed files with 1244 additions and 16 deletions
--- a/wads/archive/robots.txt
+++ b/wads/archive/robots.txt
@@ -0,0 +1,34 @@
+User-agent: sogou
+Disallow: /
+
+User-agent: *
+Disallow: /stuff/
+
+User-agent: *
+Allow: /stuff/WebQuake/
+
+User-agent: *
+Disallow: /temp/
+
+User-agent: *
+#Disallow: /webarchive/
+Disallow: /reviews/random_map.php
+Disallow: /reviews/random_recommendations.php
+
+#because he is a twat
+User-agent: bingbot
+Disallow: /46
+Disallow: /206
+Disallow: /52
+
+User-Agent: MJ12bot
+#Crawl-Delay: 5
+Disallow: /
+
+# fuck off
+User-agent: AhrefsBot 
+Disallow: /
+User-agent: SemrushBot
+Disallow: /
+User-agent: SemrushBot-SA
+Disallow: /