User-agent: *           # directed to all spiders
Disallow: /missing.html
Disallow: /failed_auth.html
Disallow: /comments.html
Disallow: /comments.cgi
Disallow: /.htaccess
Disallow: /robots.txt
Disallow: /admin/
Disallow: /angel/
Disallow: /portfolio/
Disallow: /commentsbookSRC.html