# /robots.txt file for http://webcrawler.com/
# -------------------------------------------


User-agent: *

Disallow: /_html
Disallow: /style
Disallow: /media
Disallow: /experiences
Disallow: /themes
Disallow: *.php
Disallow: *.swf