diff --git a/website/themes/beastie/layouts/robots.txt b/website/themes/beastie/layouts/robots.txt index 9bec7f76cf..1248457c75 100644 --- a/website/themes/beastie/layouts/robots.txt +++ b/website/themes/beastie/layouts/robots.txt @@ -1,16 +1,19 @@ # $FreeBSD$ User-agent: * #Crawl-delay: 10 Disallow: /cgi/getmsg.cgi Disallow: /cgi/mailindex.cgi Disallow: /cgi/man.cgi?apropos=2& Disallow: /cgi/mid.cgi Disallow: /cgi/ports.cgi Disallow: /cgi/man-autocomplete.cgi Disallow: /cgi/ports-autocomplete.cgi Disallow: /statistic User-agent: GPTBot Disallow: /cgi +User-agent: CCBot +Disallow: /cgi +