From ca4e2659497e5268abf9b17c8fd9756ee8a3347a Mon Sep 17 00:00:00 2001 From: Jan Nieuwenhuizen Date: Thu, 8 Aug 2013 08:34:12 +0200 Subject: [PATCH] Deny rogue crawler AhrefsBot. Fixes web load. The AhrefsBot is checking crawling files every second, including all binaries. This increases the load in an unacceptable way. --- Documentation/web/server/lilypond.org.htaccess | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/web/server/lilypond.org.htaccess b/Documentation/web/server/lilypond.org.htaccess index b4780a9414..5e7dfae3d9 100644 --- a/Documentation/web/server/lilypond.org.htaccess +++ b/Documentation/web/server/lilypond.org.htaccess @@ -23,6 +23,10 @@ RewriteEngine On RewriteCond %{HTTP_USER_AGENT} httrack [NC] RewriteRule ^.*/source/.*$ /please-respect-robots.txt.html [L] +# Deny rogue crawler +RewriteCond %{HTTP_USER_AGENT} ^(.*)AhrefsBot(.*) [NC] +RewriteRule .* - [F,L] + # Permanent top level entry points -- ./doc RedirectMatch ^/music-glossary /glossary RedirectMatch ^/tutorial /learning -- 2.39.2