Deny rogue crawler AhrefsBot. Fixes web load.

The AhrefsBot is checking crawling files every second, including
all binaries.  This increases the load in an unacceptable way.

diff --git a/Documentation/web/server/lilypond.org.htaccess b/Documentation/web/server/lilypond.org.htaccess
index b4780a94147df06f986cb21957f7670ec7b881a0..5e7dfae3d98773b58516b45122318f187bfd7716 100644 (file)
--- a/Documentation/web/server/lilypond.org.htaccess
+++ b/Documentation/web/server/lilypond.org.htaccess
@@ -23,6 +23,10 @@ RewriteEngine On
 RewriteCond %{HTTP_USER_AGENT} httrack [NC]
 RewriteRule ^.*/source/.*$ /please-respect-robots.txt.html [L]
 
+# Deny rogue crawler
+RewriteCond %{HTTP_USER_AGENT} ^(.*)AhrefsBot(.*) [NC]
+RewriteRule .* - [F,L]
+
 # Permanent top level entry points -- ./doc
 RedirectMatch ^/music-glossary /glossary
 RedirectMatch ^/tutorial /learning