diff --git a/website/themes/beastie/layouts/robots.txt b/website/themes/beastie/layouts/robots.txt index ac0950c19f..c8a4fa75ba 100644 --- a/website/themes/beastie/layouts/robots.txt +++ b/website/themes/beastie/layouts/robots.txt @@ -1,23 +1,14 @@ # $FreeBSD$ User-agent: * #Crawl-delay: 10 -Disallow: /cgi/confirm-code.cgi -Disallow: /cgi/cvsweb.cgi -Disallow: /cgi/dosendpr.cgi Disallow: /cgi/getmsg.cgi Disallow: /cgi/mailindex.cgi Disallow: /cgi/man.cgi?apropos=2& Disallow: /cgi/mid.cgi -Disallow: /cgi/mirror.cgi -Disallow: /cgi/missing_handler.cgi Disallow: /cgi/ports.cgi -Disallow: /cgi/query-pr-summary.cgi -Disallow: /cgi/query-pr.cgi -Disallow: /cgi/search.cgi -Disallow: /cgi/url.cgi Disallow: /statistic User-agent: GPTBot Disallow: /cgi