# This File was placed here by Josh , Please Leave it here. # Is came direct from the Microsoft Site - 120401 # This example disallows all Web Spiders for the entire site: # Make changes for all web spiders # User-agent: * # Disallow: / # The following example disallows a Robot named "WebSpider" from the virtual paths "/marketing" and "/sales": # Tell "WebSpider" where it can't go # User-agent: WebSpider # Disallow: /marketing # Disallow: /sales # Allow all other robots to browse everywhere # User-agent: * # Disallow: # This example allows only a Web Spider named "SpiderOne" into a site, while denying all other Spiders: # Allow "SpiderOne" in the site # User-agent: SpiderOne # Disallow: # Deny all other spiders # User-agent: * # Disallow: / # This last example disallows FrontPage-related paths in the root of your Web site: # Ignore FrontPage files User-agent: * Disallow: /_borders Disallow: /_derived Disallow: /_fpclass Disallow: /_overlay Disallow: /_private Disallow: /_themes Disallow: /_vti_bin Disallow: /_vti_cnf Disallow: /_vti_log Disallow: /_vti_map Disallow: /_vti_pvt Disallow: /_vti_txt Disallow: /images Disallow: /download Disallow: /jwaite # For more information on writing Robots.txt files, please see the following URL: # http: //info.webcrawler.com/mak/projects/robots/norobots.html