# Ban msnbot for being broken, badly behaved and generally
# sucking way too much bandwidth
User-agent: msnbot
Disallow: /

# Keep the webalizer logs out of the search engines
# to help stop those pesky log spammers (or at least
# it won't do them any good)
User-agent: *
Disallow: /webalizer/

User-agent: *
Disallow: /webalizer

# No point in crawling the login pages
User-agent: *
Disallow: /login/

# No point in crawling the comment submission pages either
User-agent: *
Disallow: /articles/newcomments/

# Don't crawl ~steve
User-agent: *
Disallow: /~steve/