diff --git a/website/themes/beastie/layouts/robots.txt b/website/themes/beastie/layouts/robots.txt index b3bb9e880b..70cc4362be 100644 --- a/website/themes/beastie/layouts/robots.txt +++ b/website/themes/beastie/layouts/robots.txt @@ -1,13 +1,31 @@ User-agent: * Disallow: /cgi/man.cgi?apropos=2& Disallow: /cgi/ports.cgi Disallow: /cgi/man-autocomplete.cgi Disallow: /cgi/ports-autocomplete.cgi Disallow: /statistic User-agent: GPTBot Disallow: /cgi User-agent: CCBot Disallow: /cgi +# SEO +User-agent: SemrushBot +Disallow: /cgi + +User-agent: MJ12bot +Disallow: /cgi + +User-agent: BLEXBot +Disallow: /cgi + +User-agent: dotbot +Disallow: /cgi + +User-agent: SeekportBot +Disallow: /cgi + +User-agent: AhrefsBot +Disallow: /cgi