From e7d5323344e0bbd601070d204cc5feb38928f323 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Thu, 12 Dec 2019 06:03:38 -0700 Subject: [PATCH] htaccess: recognize more crawlers Signed-off-by: Kyle J. McKay --- cgi/htaccess | 6 +++--- htaccess | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cgi/htaccess b/cgi/htaccess index 4678740..7408eb5 100644 --- a/cgi/htaccess +++ b/cgi/htaccess @@ -4,14 +4,14 @@ RewriteEngine On RewriteBase / -RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71) [NC] +RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71|Crawl) [NC] RewriteCond %{QUERY_STRING} a=(blame|blame_data|blame_incremental|blob_plain|blobdiff|diff|snapshot|commitdiff|history|refs|shortlog) [OR] RewriteCond %{QUERY_STRING} a=log.*f= RewriteRule .* - [NS,F,L] -RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71) [NC] +RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71|Crawl) [NC] RewriteRule /(blame|blame_data|blame_incremental|blob_plain|blobdiff|bundles|diff|snapshot|commitdiff|history|refs|shortlog)(/|$) - [NS,F,L] -RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71) [NC] +RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71|Crawl) [NC] RewriteCond %{QUERY_STRING} (.*h=.*;)hb=[^;]*(.*) RewriteRule (.*) $1?%1%2 [NS,R=303,NE,L] diff --git a/htaccess b/htaccess index 759ef2f..a8805a2 100644 --- a/htaccess +++ b/htaccess @@ -4,7 +4,7 @@ RewriteEngine On RewriteBase / -RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71) [NC] +RewriteCond %{HTTP_USER_AGENT} (Slurp|Bot|Spider|Riddler|ltx71|Crawl) [NC] RewriteRule ^r/ - [NS,F,L] # If you have mod_status enabled AND this host ends up being the default -- 2.11.4.GIT