# Robots.txt file from http://www.searchengineworld.com # # Built from text file # http://info.webcrawler.com/mak/projects/robots/active/all.txt # # This restricts access to only known and registered robots. # User-agent: BecomeBot Disallow: / User-agent: twiceler Disallow: / User-agent: YodaoBot Disallow: / User-agent: * Disallow: /cgi-bin/ Disallow: /webstats/ Disallow: /includes/ Disallow: /files/index.php?action=report* Disallow: /files/index.php?action=upload* Disallow: /files/index.php?action=submit* Disallow: /files/index.php?action=download* Disallow: /boards/index.php?action=login* Disallow: /boards/index.php?action=register* Disallow: /boards/index.php?action=reminder* Disallow: /boards/index.php?action=printpage* Disallow: /boards/index.php?action=profile* Disallow: /boards/index.php?action=emailuser* Disallow: /boards/index.php?action=help* Disallow: /boards/index.php?action=search* Disallow: /boards/index.php?action=viewers* Disallow: /boards/index.php?action=unread* Disallow: /boards/index.php?action=verificationcode*