# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # robots.txt for http://www.hirethings.co.nz User-agent: * Crawl-delay: 3 # yahoo specific for the Slurp crawler Disallow: / #Disallow: /myprofile/* #Disallow: /contact/list/* #Disallow: /loan/* #Disallow: /booking/* #Disallow: /use_request/* #Disallow: /picker/* #Disallow: /search/* #Disallow: /javascripts/* #Disallow: /home/* #Disallow: /category/select_list/* #Disallow: /payment/*